From 4d91bf2bda0d51f5b328a9a81cf4dd91c7f1c6c2 Mon Sep 17 00:00:00 2001 From: sentriz Date: Tue, 4 Jun 2019 15:27:37 +0100 Subject: [PATCH] merge album and folder models --- go.mod | 3 + go.sum | 6 ++ model/model.go | 117 +++++++++---------------- scanner/folder_stack.go | 29 ++++--- scanner/scanner.go | 47 +++++------ scanner/utilities.go | 20 +---- scanner/walk.go | 183 ++++++++++++++++------------------------ 7 files changed, 164 insertions(+), 241 deletions(-) diff --git a/go.mod b/go.mod index a9b5351..a4d9686 100644 --- a/go.mod +++ b/go.mod @@ -28,4 +28,7 @@ require ( golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b // indirect google.golang.org/appengine v1.5.0 // indirect + gopkg.in/axiomzen/null.v3 v3.2.4 + gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect + gopkg.in/pg.v4 v4.9.5 // indirect ) diff --git a/go.sum b/go.sum index a9bc273..c74e204 100644 --- a/go.sum +++ b/go.sum @@ -263,10 +263,16 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/axiomzen/null.v3 v3.2.4 h1:5VmJ9lSU0dBJjisXhuhRnGiIolhTjkmQQ0EBNE9Z5QY= +gopkg.in/axiomzen/null.v3 v3.2.4/go.mod h1:Vq8/79AVvSZVg5PdN4kNROnTff7WtSh0HOiBHfOvVNU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/pg.v4 v4.9.5 h1:bs21aaMPPPcUPhNtqGxN8EeYUFU10MsNrC7U9m/lJgU= +gopkg.in/pg.v4 v4.9.5/go.mod h1:cSUPtzgofjgARAbFCE5u6WDHGPgbR1sjUYcWQlKvpec= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/model/model.go b/model/model.go index 62ea379..09310f3 100644 --- a/model/model.go +++ b/model/model.go @@ -1,6 +1,8 @@ package model -import "time" +import ( + "time" +) // q: what in tarnation are the `IsNew`s for? // a: it's a bit of a hack - but we set a models IsNew to true if @@ -9,108 +11,73 @@ import "time" // that bool being true - since it won't be true if it was already // in the db -// Album represents the albums table -type Album struct { - IDBase - CrudBase - Artist Artist - ArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` - Title string `gorm:"not null; index"` - // an Album having a `Path` is a little weird when browsing by tags - // (for the most part - the library's folder structure is treated as - // if it were flat), but this solves the "American Football problem" - // https://en.wikipedia.org/wiki/American_Football_(band)#Discography - Path string `gorm:"not null; unique_index"` - CoverID int `sql:"default: null; type:int REFERENCES covers(id)"` - Cover Cover - Year int - Tracks []Track - IsNew bool `gorm:"-"` -} - -// Artist represents the Artists table type Artist struct { IDBase CrudBase - Name string `gorm:"not null; unique_index"` - Albums []Album + Name string `gorm:"not null; unique_index"` + Folders []Folder } -// Track represents the tracks table type Track struct { IDBase CrudBase - Album Album - AlbumID int `gorm:"index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` - Artist Artist - ArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` - TrackArtist string - Bitrate int - Codec string - DiscNumber int - Duration int - Title string - TotalDiscs int - TotalTracks int - TrackNumber int - Year int - Suffix string - ContentType string - Size int - Folder Folder - FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` - Path string `gorm:"not null; unique_index"` + Folder Folder + // TODO: try removing idx_folder_basename_ext + FolderID int `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` + Filename string `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null"` + Ext string `gorm:"not nill; unique_index:idx_folder_filename_ext" sql:"default: null"` + Artist Artist + ArtistID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` + ContentType string `gorm:"not null" sql:"default: null"` + Duration int `gorm:"not null" sql:"default: null"` + Size int `gorm:"not null" sql:"default: null"` + Bitrate int `gorm:"not null" sql:"default: null"` + TagDiscNumber int `sql:"default: null"` + TagTitle string `sql:"default: null"` + TagTotalDiscs int `sql:"default: null"` + TagTotalTracks int `sql:"default: null"` + TagTrackArtist string `sql:"default: null"` + TagTrackNumber int `sql:"default: null"` + TagYear int `sql:"default: null"` } -// Cover represents the covers table -type Cover struct { - IDBase - CrudBase - Image []byte - Path string `gorm:"not null; unique_index"` - IsNew bool `gorm:"-"` -} - -// User represents the users table type User struct { IDBase CrudBase - Name string `gorm:"not null; unique_index"` - Password string - LastFMSession string - IsAdmin bool + Name string `gorm:"not null; unique_index" sql:"default: null"` + Password string `gorm:"not null" sql:"default: null"` + LastFMSession string `sql:"default: null"` + IsAdmin bool `sql:"default: null"` } -// Setting represents the settings table type Setting struct { CrudBase - Key string `gorm:"primary_key; auto_increment:false"` - Value string + Key string `gorm:"not null; primary_key; auto_increment:false" sql:"default: null"` + Value string `sql:"default: null"` } -// Play represents the settings table type Play struct { IDBase User User UserID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"` - Album Album - AlbumID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` Folder Folder - FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` - Time time.Time + FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` + Time time.Time `sql:"default: null"` Count int } -// Folder represents the settings table type Folder struct { IDBase CrudBase - Name string - Path string `gorm:"not null; unique_index"` - Parent *Folder - ParentID int `sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` - CoverID int `sql:"default: null; type:int REFERENCES covers(id)"` - HasTracks bool `gorm:"not null; index"` - Cover Cover - IsNew bool `gorm:"-"` + LeftPath string `gorm:"unique_index:idx_left_path_right_path"` + RightPath string `gorm:"not null; unique_index:idx_left_path_right_path" sql:"default: null"` + Parent *Folder + ParentID int `sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"` + AlbumArtist Artist + AlbumArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` + AlbumTitle string `gorm:"index" sql:"default: null"` + AlbumYear int `sql:"default: null"` + Cover string `sql:"default: null"` + Tracks []Track + IsNew bool `gorm:"-"` } diff --git a/scanner/folder_stack.go b/scanner/folder_stack.go index d58585d..26f4f5d 100644 --- a/scanner/folder_stack.go +++ b/scanner/folder_stack.go @@ -1,35 +1,40 @@ package scanner -import "github.com/sentriz/gonic/model" +import ( + "fmt" + "strings" -type folderStack []model.Folder + "github.com/sentriz/gonic/model" +) -func (s *folderStack) Push(v model.Folder) { +type folderStack []*model.Folder + +func (s *folderStack) Push(v *model.Folder) { *s = append(*s, v) } -func (s *folderStack) Pop() model.Folder { +func (s *folderStack) Pop() *model.Folder { l := len(*s) if l == 0 { - return model.Folder{} + return nil } r := (*s)[l-1] *s = (*s)[:l-1] return r } -func (s *folderStack) Peek() model.Folder { +func (s *folderStack) Peek() *model.Folder { l := len(*s) if l == 0 { - return model.Folder{} + return nil } return (*s)[l-1] } -func (s *folderStack) PeekID() int { - l := len(*s) - if l == 0 { - return 0 +func (s *folderStack) String() string { + paths := make([]string, len(*s)) + for i, folder := range *s { + paths[i] = folder.RightPath } - return (*s)[l-1].ID + return fmt.Sprintf("[%s]", strings.Join(paths, " ")) } diff --git a/scanner/scanner.go b/scanner/scanner.go index b82f5ea..b817803 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -1,12 +1,5 @@ package scanner -// Album -> needs a CoverID -// -> needs a FolderID (American Football) -// Folder -> needs a CoverID -// -> needs a ParentID -// Track -> needs an AlbumID -// -> needs a FolderID - import ( "log" "sync/atomic" @@ -26,27 +19,32 @@ var ( type Scanner struct { db, tx *gorm.DB musicPath string - seenTracks map[string]bool + seenTracks map[int]struct{} curFolders folderStack - curTracks []model.Track - curCover model.Cover - curAlbum model.Album - curAArtist model.Artist + curCover string } func New(db *gorm.DB, musicPath string) *Scanner { return &Scanner{ db: db, musicPath: musicPath, - seenTracks: make(map[string]bool), + seenTracks: make(map[int]struct{}), curFolders: make(folderStack, 0), - curTracks: make([]model.Track, 0), - curCover: model.Cover{}, - curAlbum: model.Album{}, - curAArtist: model.Artist{}, } } +func (s *Scanner) curFolder() *model.Folder { + return s.curFolders.Peek() +} + +func (s *Scanner) curFolderID() int { + peek := s.curFolders.Peek() + if peek == nil { + return 0 + } + return peek.ID +} + func (s *Scanner) Start() error { if atomic.LoadInt32(&IsScanning) == 1 { return errors.New("already scanning") @@ -86,16 +84,17 @@ func (s *Scanner) startClean() error { defer logElapsed(time.Now(), "cleaning database") var tracks []model.Track s.tx. - Select("id, path"). + Select("id"). Find(&tracks) + var deleted int for _, track := range tracks { - _, ok := s.seenTracks[track.Path] - if ok { - continue + _, ok := s.seenTracks[track.ID] + if !ok { + s.tx.Delete(&track) + deleted++ } - s.tx.Delete(&track) - log.Println("removed track", track.Path) } + log.Printf("removed %d tracks\n", deleted) return nil } @@ -104,10 +103,8 @@ func (s *Scanner) MigrateDB() error { s.tx = s.db.Begin() defer s.tx.Commit() s.tx.AutoMigrate( - model.Album{}, model.Artist{}, model.Track{}, - model.Cover{}, model.User{}, model.Setting{}, model.Play{}, diff --git a/scanner/utilities.go b/scanner/utilities.go index 682bcf8..70f88a2 100644 --- a/scanner/utilities.go +++ b/scanner/utilities.go @@ -2,15 +2,12 @@ package scanner import ( "os" - "path" - "path/filepath" - "strings" "github.com/dhowden/tag" "github.com/pkg/errors" ) -var trackExtensions = map[string]string{ +var mimeTypes = map[string]string{ "mp3": "audio/mpeg", "flac": "audio/x-flac", "aac": "audio/x-aac", @@ -18,15 +15,6 @@ var trackExtensions = map[string]string{ "ogg": "audio/ogg", } -func isTrack(fullPath string) (string, string, bool) { - ext := filepath.Ext(fullPath)[1:] - mine, ok := trackExtensions[ext] - if !ok { - return "", "", false - } - return mine, ext, true -} - var coverFilenames = map[string]struct{}{ "cover.png": struct{}{}, "cover.jpg": struct{}{}, @@ -42,12 +30,6 @@ var coverFilenames = map[string]struct{}{ "front.jpeg": struct{}{}, } -func isCover(fullPath string) bool { - _, filename := path.Split(fullPath) - _, ok := coverFilenames[strings.ToLower(filename)] - return ok -} - func readTags(path string) (tag.Metadata, error) { trackData, err := os.Open(path) if err != nil { diff --git a/scanner/walk.go b/scanner/walk.go index fb15d75..ac85627 100644 --- a/scanner/walk.go +++ b/scanner/walk.go @@ -1,7 +1,6 @@ package scanner import ( - "io/ioutil" "log" "os" "path" @@ -14,182 +13,146 @@ import ( "github.com/sentriz/gonic/model" ) -type trackItem struct { - mime string - ext string -} - type item struct { - path string - relPath string - stat os.FileInfo - track *trackItem + // + // common + fullPath string + relPath string + directory string + filename string + stat os.FileInfo + // + // track only + ext string + mime string } -func (s *Scanner) callbackItem(path string, info *godirwalk.Dirent) error { - stat, err := os.Stat(path) +func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error { + stat, err := os.Stat(fullPath) if err != nil { return errors.Wrap(err, "stating") } - relPath, err := filepath.Rel(s.musicPath, path) + relPath, err := filepath.Rel(s.musicPath, fullPath) if err != nil { return errors.Wrap(err, "getting relative path") } + directory, filename := path.Split(relPath) it := &item{ - path: path, - relPath: relPath, - stat: stat, + fullPath: fullPath, + relPath: relPath, + directory: directory, + filename: filename, + stat: stat, } if info.IsDir() { return s.handleFolder(it) } - if isCover(path) { - return s.handleCover(it) + if _, ok := coverFilenames[filename]; ok { + s.curCover = filename + return nil } - if mime, ext, ok := isTrack(path); ok { - s.seenTracks[relPath] = true - it.track = &trackItem{mime: mime, ext: ext} + ext := path.Ext(filename)[1:] + if mime, ok := mimeTypes[ext]; ok { + it.ext = ext + it.mime = mime return s.handleTrack(it) } return nil } -func (s *Scanner) callbackPost(path string, info *godirwalk.Dirent) error { - // in general in this function - if a model is not nil, then it - // has at least been looked up. if it has a id of 0, then it is - // a new record and needs to be inserted - if s.curCover.IsNew { - s.tx.Save(&s.curCover) - } - if s.curAlbum.IsNew { - s.curAlbum.CoverID = s.curCover.ID - s.tx.Save(&s.curAlbum) - } +func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error { folder := s.curFolders.Pop() if folder.IsNew { - folder.ParentID = s.curFolders.PeekID() - folder.CoverID = s.curCover.ID - folder.HasTracks = len(s.curTracks) > 1 + folder.ParentID = s.curFolderID() + folder.Cover = s.curCover s.tx.Save(&folder) } - for _, t := range s.curTracks { - t.FolderID = folder.ID - t.AlbumID = s.curAlbum.ID - s.tx.Save(&t) - } - // - s.curTracks = make([]model.Track, 0) - s.curCover = model.Cover{} - s.curAlbum = model.Album{} - s.curAArtist = model.Artist{} - // - log.Printf("processed folder `%s`\n", path) + s.curCover = "" + log.Printf("processed folder `%s`\n", fullPath) return nil } func (s *Scanner) handleFolder(it *item) error { - // TODO: var folder model.Folder err := s.tx. - Where("path = ?", it.relPath). + Where(model.Folder{ + LeftPath: it.directory, + RightPath: it.filename, + }). First(&folder). Error if !gorm.IsRecordNotFoundError(err) && it.stat.ModTime().Before(folder.UpdatedAt) { // we found the record but it hasn't changed - s.curFolders.Push(folder) + s.curFolders.Push(&folder) return nil } - folder.Path = it.relPath - folder.Name = it.stat.Name() + folder.LeftPath = it.directory + folder.RightPath = it.filename s.tx.Save(&folder) folder.IsNew = true - s.curFolders.Push(folder) - return nil -} - -func (s *Scanner) handleCover(it *item) error { - err := s.tx. - Where("path = ?", it.relPath). - First(&s.curCover). - Error - if !gorm.IsRecordNotFoundError(err) && - it.stat.ModTime().Before(s.curCover.UpdatedAt) { - // we found the record but it hasn't changed - return nil - } - s.curCover.Path = it.relPath - image, err := ioutil.ReadFile(it.path) - if err != nil { - return errors.Wrap(err, "reading cover") - } - s.curCover.Image = image - s.curCover.IsNew = true + s.curFolders.Push(&folder) return nil } func (s *Scanner) handleTrack(it *item) error { // // set track basics - track := model.Track{} + var track model.Track err := s.tx. - Where("path = ?", it.relPath). + Where(model.Track{ + FolderID: s.curFolderID(), + Filename: it.filename, + Ext: it.ext, + }). First(&track). Error if !gorm.IsRecordNotFoundError(err) && it.stat.ModTime().Before(track.UpdatedAt) { + s.seenTracks[track.ID] = struct{}{} // we found the record but it hasn't changed return nil } - tags, err := readTags(it.path) + track.Filename = it.filename + track.Ext = it.ext + track.ContentType = it.mime + track.Size = int(it.stat.Size()) + track.FolderID = s.curFolderID() + track.Duration = -1 + track.Bitrate = -1 + tags, err := readTags(it.fullPath) if err != nil { return errors.Wrap(err, "reading tags") } trackNumber, totalTracks := tags.Track() discNumber, totalDiscs := tags.Disc() - track.DiscNumber = discNumber - track.TotalDiscs = totalDiscs - track.TotalTracks = totalTracks - track.TrackNumber = trackNumber - track.Path = it.relPath - track.Suffix = it.track.ext - track.ContentType = it.track.mime - track.Size = int(it.stat.Size()) - track.Title = tags.Title() - track.TrackArtist = tags.Artist() - track.Year = tags.Year() - track.FolderID = s.curFolders.PeekID() + track.TagDiscNumber = discNumber + track.TagTotalDiscs = totalDiscs + track.TagTotalTracks = totalTracks + track.TagTrackNumber = trackNumber + track.TagTitle = tags.Title() + track.TagTrackArtist = tags.Artist() + track.TagYear = tags.Year() // // set album artist basics + var artist model.Artist err = s.tx.Where("name = ?", tags.AlbumArtist()). - First(&s.curAArtist). + First(&artist). Error if gorm.IsRecordNotFoundError(err) { - s.curAArtist.Name = tags.AlbumArtist() - s.tx.Save(&s.curAArtist) + artist.Name = tags.AlbumArtist() + s.tx.Save(&artist) } - track.ArtistID = s.curAArtist.ID + track.ArtistID = artist.ID + s.tx.Save(&track) + s.seenTracks[track.ID] = struct{}{} // // set album if this is the first track in the folder - if len(s.curTracks) > 0 { - s.curTracks = append(s.curTracks, track) + if !s.curFolder().IsNew { return nil } - s.curTracks = append(s.curTracks, track) - // - directory, _ := path.Split(it.relPath) - err = s.tx. - Where("path = ?", directory). - First(&s.curAlbum). - Error - if !gorm.IsRecordNotFoundError(err) { - // we found the record - return nil - } - s.curAlbum.Path = directory - s.curAlbum.Title = tags.Album() - s.curAlbum.Year = tags.Year() - s.curAlbum.ArtistID = s.curAArtist.ID - s.curAlbum.IsNew = true + s.curFolder().AlbumTitle = tags.Album() + s.curFolder().AlbumYear = tags.Year() + s.curFolder().AlbumArtistID = artist.ID return nil }