diff --git a/cmd/scanner/main.go b/cmd/scanner/main.go index 7096a1b..fb9f76b 100644 --- a/cmd/scanner/main.go +++ b/cmd/scanner/main.go @@ -37,6 +37,7 @@ func main() { log.Fatalf("error opening database: %v\n", err) } db.SetLogger(log.New(os.Stdout, "gorm ", 0)) + db.Exec("PRAGMA foreign_keys = ON") s := scanner.New( db, *musicPath, diff --git a/cmd/server/main.go b/cmd/server/main.go index f9186eb..ee5c99d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -46,6 +46,7 @@ func main() { if err != nil { log.Fatalf("error opening database: %v\n", err) } + db.Exec("PRAGMA foreign_keys = ON") s := server.New( db, *musicPath, diff --git a/scanner/scanner.go b/scanner/scanner.go index a4bdce6..7b0f6fc 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -1,11 +1,5 @@ package scanner -// this scanner tries to scan with a single unsorted walk of the music -// directory - which means you can come across the cover of an album/folder -// before the tracks (and therefore the album) which is an issue because -// when inserting into the album table, we need a reference to the cover. -// to solve this we're using godirwalk's PostChildrenCallback and some globals -// // Album -> needs a CoverID // -> needs a FolderID (American Football) // Folder -> needs a CoverID @@ -14,11 +8,7 @@ package scanner // -> needs a FolderID import ( - "io/ioutil" "log" - "os" - "path" - "path/filepath" "sync/atomic" "time" @@ -33,18 +23,6 @@ var ( IsScanning int32 ) -type trackItem struct { - mime string - ext string -} - -type item struct { - path string - relPath string - stat os.FileInfo - track *trackItem -} - type Scanner struct { db, tx *gorm.DB musicPath string @@ -69,179 +47,33 @@ func New(db *gorm.DB, musicPath string) *Scanner { } } -func (s *Scanner) handleCover(it *item) error { - err := s.tx. - Where("path = ?", it.relPath). - First(&s.curCover). - Error - if !gorm.IsRecordNotFoundError(err) && - it.stat.ModTime().Before(s.curCover.UpdatedAt) { - // we found the record but it hasn't changed - return nil +func (s *Scanner) Start() error { + if atomic.LoadInt32(&IsScanning) == 1 { + return errors.New("already scanning") } - s.curCover.Path = it.relPath - image, err := ioutil.ReadFile(it.path) - if err != nil { - return errors.Wrap(err, "reading cover") + atomic.StoreInt32(&IsScanning, 1) + defer atomic.StoreInt32(&IsScanning, 0) + s.tx = s.db.Begin() + defer s.tx.Commit() + if err := s.startScan(); err != nil { + return errors.Wrap(err, "start scan") + } + if err := s.startClean(); err != nil { + return errors.Wrap(err, "start clean") } - s.curCover.Image = image - s.curCover.IsNew = true return nil } -func (s *Scanner) handleFolder(it *item) error { - // TODO: - var folder model.Folder - err := s.tx. - Where("path = ?", it.relPath). - First(&folder). - Error - if !gorm.IsRecordNotFoundError(err) && - it.stat.ModTime().Before(folder.UpdatedAt) { - // we found the record but it hasn't changed - s.curFolders.Push(folder) - return nil - } - folder.Path = it.relPath - folder.Name = it.stat.Name() - s.tx.Save(&folder) - folder.IsNew = true - s.curFolders.Push(folder) - return nil -} - -func (s *Scanner) handleTrack(it *item) error { - // - // set track basics - track := model.Track{} - err := s.tx. - Where("path = ?", it.relPath). - First(&track). - Error - if !gorm.IsRecordNotFoundError(err) && - it.stat.ModTime().Before(track.UpdatedAt) { - // we found the record but it hasn't changed - return nil - } - tags, err := readTags(it.path) - if err != nil { - return errors.Wrap(err, "reading tags") - } - trackNumber, totalTracks := tags.Track() - discNumber, totalDiscs := tags.Disc() - track.DiscNumber = discNumber - track.TotalDiscs = totalDiscs - track.TotalTracks = totalTracks - track.TrackNumber = trackNumber - track.Path = it.relPath - track.Suffix = it.track.ext - track.ContentType = it.track.mime - track.Size = int(it.stat.Size()) - track.Title = tags.Title() - track.Artist = tags.Artist() - track.Year = tags.Year() - track.FolderID = s.curFolders.PeekID() - // - // set album artist basics - err = s.tx.Where("name = ?", tags.AlbumArtist()). - First(&s.curAArtist). - Error - if gorm.IsRecordNotFoundError(err) { - s.curAArtist.Name = tags.AlbumArtist() - s.tx.Save(&s.curAArtist) - } - track.AlbumArtistID = s.curAArtist.ID - // - // set album if this is the first track in the folder - if len(s.curTracks) > 0 { - s.curTracks = append(s.curTracks, track) - return nil - } - s.curTracks = append(s.curTracks, track) - // - directory, _ := path.Split(it.relPath) - err = s.tx. - Where("path = ?", directory). - First(&s.curAlbum). - Error - if !gorm.IsRecordNotFoundError(err) { - // we found the record - return nil - } - s.curAlbum.Path = directory - s.curAlbum.Title = tags.Album() - s.curAlbum.Year = tags.Year() - s.curAlbum.AlbumArtistID = s.curAArtist.ID - s.curAlbum.IsNew = true - return nil -} - -func (s *Scanner) handleFolderCompletion(path string, info *godirwalk.Dirent) error { - // in general in this function - if a model is not nil, then it - // has at least been looked up. if it has a id of 0, then it is - // a new record and needs to be inserted - if s.curCover.IsNew { - s.tx.Save(&s.curCover) - } - if s.curAlbum.IsNew { - s.curAlbum.CoverID = s.curCover.ID - s.tx.Save(&s.curAlbum) - } - folder := s.curFolders.Pop() - if folder.IsNew { - folder.ParentID = s.curFolders.PeekID() - folder.CoverID = s.curCover.ID - folder.HasTracks = len(s.curTracks) > 1 - s.tx.Save(&folder) - } - for _, t := range s.curTracks { - t.FolderID = folder.ID - t.AlbumID = s.curAlbum.ID - s.tx.Save(&t) - } - // - s.curTracks = make([]model.Track, 0) - s.curCover = model.Cover{} - s.curAlbum = model.Album{} - s.curAArtist = model.AlbumArtist{} - // - log.Printf("processed folder `%s`\n", path) - return nil -} - -func (s *Scanner) handleItem(path string, info *godirwalk.Dirent) error { - stat, err := os.Stat(path) - if err != nil { - return errors.Wrap(err, "stating") - } - relPath, err := filepath.Rel(s.musicPath, path) - if err != nil { - return errors.Wrap(err, "getting relative path") - } - it := &item{ - path: path, - relPath: relPath, - stat: stat, - } - if info.IsDir() { - return s.handleFolder(it) - } - if isCover(path) { - return s.handleCover(it) - } - if mime, ext, ok := isTrack(path); ok { - s.seenTracks[relPath] = true - it.track = &trackItem{mime: mime, ext: ext} - return s.handleTrack(it) - } - return nil +func logElapsed(start time.Time, name string) { + elapsed := time.Since(start) + log.Printf("finished %s in %s\n", name, elapsed) } func (s *Scanner) startScan() error { defer logElapsed(time.Now(), "scanning") err := godirwalk.Walk(s.musicPath, &godirwalk.Options{ - Callback: s.handleItem, - PostChildrenCallback: s.handleFolderCompletion, + Callback: s.callbackItem, + PostChildrenCallback: s.callbackPost, Unsorted: true, }) if err != nil { @@ -269,7 +101,6 @@ func (s *Scanner) startClean() error { func (s *Scanner) MigrateDB() error { defer logElapsed(time.Now(), "migrating database") - s.db.Exec("PRAGMA foreign_keys = ON") s.tx = s.db.Begin() defer s.tx.Commit() s.tx.AutoMigrate( @@ -289,21 +120,3 @@ func (s *Scanner) MigrateDB() error { }) return nil } - -func (s *Scanner) Start() error { - if atomic.LoadInt32(&IsScanning) == 1 { - return errors.New("already scanning") - } - atomic.StoreInt32(&IsScanning, 1) - defer atomic.StoreInt32(&IsScanning, 0) - s.db.Exec("PRAGMA foreign_keys = ON") - s.tx = s.db.Begin() - defer s.tx.Commit() - if err := s.startScan(); err != nil { - return errors.Wrap(err, "start scan") - } - if err := s.startClean(); err != nil { - return errors.Wrap(err, "start clean") - } - return nil -} diff --git a/scanner/utilities.go b/scanner/utilities.go index 4318f9b..c52da83 100644 --- a/scanner/utilities.go +++ b/scanner/utilities.go @@ -2,12 +2,10 @@ package scanner import ( "fmt" - "log" "os" "path" "path/filepath" "strings" - "time" "github.com/dhowden/tag" ) @@ -62,8 +60,3 @@ func readTags(path string) (tag.Metadata, error) { } return tags, nil } - -func logElapsed(start time.Time, name string) { - elapsed := time.Since(start) - log.Printf("finished %s in %s\n", name, elapsed) -} diff --git a/scanner/walk.go b/scanner/walk.go new file mode 100644 index 0000000..a759772 --- /dev/null +++ b/scanner/walk.go @@ -0,0 +1,195 @@ +package scanner + +import ( + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + + "github.com/jinzhu/gorm" + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + + "github.com/sentriz/gonic/model" +) + +type trackItem struct { + mime string + ext string +} + +type item struct { + path string + relPath string + stat os.FileInfo + track *trackItem +} + +func (s *Scanner) callbackItem(path string, info *godirwalk.Dirent) error { + stat, err := os.Stat(path) + if err != nil { + return errors.Wrap(err, "stating") + } + relPath, err := filepath.Rel(s.musicPath, path) + if err != nil { + return errors.Wrap(err, "getting relative path") + } + it := &item{ + path: path, + relPath: relPath, + stat: stat, + } + if info.IsDir() { + return s.handleFolder(it) + } + if isCover(path) { + return s.handleCover(it) + } + if mime, ext, ok := isTrack(path); ok { + s.seenTracks[relPath] = true + it.track = &trackItem{mime: mime, ext: ext} + return s.handleTrack(it) + } + return nil +} + +func (s *Scanner) callbackPost(path string, info *godirwalk.Dirent) error { + // in general in this function - if a model is not nil, then it + // has at least been looked up. if it has a id of 0, then it is + // a new record and needs to be inserted + if s.curCover.IsNew { + s.tx.Save(&s.curCover) + } + if s.curAlbum.IsNew { + s.curAlbum.CoverID = s.curCover.ID + s.tx.Save(&s.curAlbum) + } + folder := s.curFolders.Pop() + if folder.IsNew { + folder.ParentID = s.curFolders.PeekID() + folder.CoverID = s.curCover.ID + folder.HasTracks = len(s.curTracks) > 1 + s.tx.Save(&folder) + } + for _, t := range s.curTracks { + t.FolderID = folder.ID + t.AlbumID = s.curAlbum.ID + s.tx.Save(&t) + } + // + s.curTracks = make([]model.Track, 0) + s.curCover = model.Cover{} + s.curAlbum = model.Album{} + s.curAArtist = model.AlbumArtist{} + // + log.Printf("processed folder `%s`\n", path) + return nil +} + +func (s *Scanner) handleFolder(it *item) error { + // TODO: + var folder model.Folder + err := s.tx. + Where("path = ?", it.relPath). + First(&folder). + Error + if !gorm.IsRecordNotFoundError(err) && + it.stat.ModTime().Before(folder.UpdatedAt) { + // we found the record but it hasn't changed + s.curFolders.Push(folder) + return nil + } + folder.Path = it.relPath + folder.Name = it.stat.Name() + s.tx.Save(&folder) + folder.IsNew = true + s.curFolders.Push(folder) + return nil +} + +func (s *Scanner) handleCover(it *item) error { + err := s.tx. + Where("path = ?", it.relPath). + First(&s.curCover). + Error + if !gorm.IsRecordNotFoundError(err) && + it.stat.ModTime().Before(s.curCover.UpdatedAt) { + // we found the record but it hasn't changed + return nil + } + s.curCover.Path = it.relPath + image, err := ioutil.ReadFile(it.path) + if err != nil { + return errors.Wrap(err, "reading cover") + } + s.curCover.Image = image + s.curCover.IsNew = true + return nil +} + +func (s *Scanner) handleTrack(it *item) error { + // + // set track basics + track := model.Track{} + err := s.tx. + Where("path = ?", it.relPath). + First(&track). + Error + if !gorm.IsRecordNotFoundError(err) && + it.stat.ModTime().Before(track.UpdatedAt) { + // we found the record but it hasn't changed + return nil + } + tags, err := readTags(it.path) + if err != nil { + return errors.Wrap(err, "reading tags") + } + trackNumber, totalTracks := tags.Track() + discNumber, totalDiscs := tags.Disc() + track.DiscNumber = discNumber + track.TotalDiscs = totalDiscs + track.TotalTracks = totalTracks + track.TrackNumber = trackNumber + track.Path = it.relPath + track.Suffix = it.track.ext + track.ContentType = it.track.mime + track.Size = int(it.stat.Size()) + track.Title = tags.Title() + track.Artist = tags.Artist() + track.Year = tags.Year() + track.FolderID = s.curFolders.PeekID() + // + // set album artist basics + err = s.tx.Where("name = ?", tags.AlbumArtist()). + First(&s.curAArtist). + Error + if gorm.IsRecordNotFoundError(err) { + s.curAArtist.Name = tags.AlbumArtist() + s.tx.Save(&s.curAArtist) + } + track.AlbumArtistID = s.curAArtist.ID + // + // set album if this is the first track in the folder + if len(s.curTracks) > 0 { + s.curTracks = append(s.curTracks, track) + return nil + } + s.curTracks = append(s.curTracks, track) + // + directory, _ := path.Split(it.relPath) + err = s.tx. + Where("path = ?", directory). + First(&s.curAlbum). + Error + if !gorm.IsRecordNotFoundError(err) { + // we found the record + return nil + } + s.curAlbum.Path = directory + s.curAlbum.Title = tags.Album() + s.curAlbum.Year = tags.Year() + s.curAlbum.AlbumArtistID = s.curAArtist.ID + s.curAlbum.IsNew = true + return nil +}