restructure scanner

cmd/scanner/main.go

@@ -37,6 +37,7 @@ func main() {
 		log.Fatalf("error opening database: %v\n", err)
 	}
 	db.SetLogger(log.New(os.Stdout, "gorm ", 0))
+	db.Exec("PRAGMA foreign_keys = ON")
 	s := scanner.New(
 		db,
 		*musicPath,

cmd/server/main.go

@@ -46,6 +46,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("error opening database: %v\n", err)
 	}
+	db.Exec("PRAGMA foreign_keys = ON")
 	s := server.New(
 		db,
 		*musicPath,

scanner/scanner.go

@@ -1,11 +1,5 @@
 package scanner
 
-// this scanner tries to scan with a single unsorted walk of the music
-// directory - which means you can come across the cover of an album/folder
-// before the tracks (and therefore the album) which is an issue because
-// when inserting into the album table, we need a reference to the cover.
-// to solve this we're using godirwalk's PostChildrenCallback and some globals
-//
 // Album  -> needs a  CoverID
 //        -> needs a  FolderID (American Football)
 // Folder -> needs a  CoverID
@@ -14,11 +8,7 @@ package scanner
 //        -> needs a  FolderID
 
 import (
-	"io/ioutil"
 	"log"
-	"os"
-	"path"
-	"path/filepath"
 	"sync/atomic"
 	"time"
 
@@ -33,18 +23,6 @@ var (
 	IsScanning int32
 )
 
-type trackItem struct {
-	mime string
-	ext  string
-}
-
-type item struct {
-	path    string
-	relPath string
-	stat    os.FileInfo
-	track   *trackItem
-}
-
 type Scanner struct {
 	db, tx     *gorm.DB
 	musicPath  string
@@ -69,179 +47,33 @@ func New(db *gorm.DB, musicPath string) *Scanner {
 	}
 }
 
-func (s *Scanner) handleCover(it *item) error {
+func (s *Scanner) Start() error {
-	err := s.tx.
+	if atomic.LoadInt32(&IsScanning) == 1 {
-		Where("path = ?", it.relPath).
+		return errors.New("already scanning")
-		First(&s.curCover).
-		Error
-	if !gorm.IsRecordNotFoundError(err) &&
-		it.stat.ModTime().Before(s.curCover.UpdatedAt) {
-		// we found the record but it hasn't changed
-		return nil
 	}
-	s.curCover.Path = it.relPath
+	atomic.StoreInt32(&IsScanning, 1)
-	image, err := ioutil.ReadFile(it.path)
+	defer atomic.StoreInt32(&IsScanning, 0)
-	if err != nil {
+	s.tx = s.db.Begin()
-		return errors.Wrap(err, "reading cover")
+	defer s.tx.Commit()
+	if err := s.startScan(); err != nil {
+		return errors.Wrap(err, "start scan")
+	}
+	if err := s.startClean(); err != nil {
+		return errors.Wrap(err, "start clean")
 	}
-	s.curCover.Image = image
-	s.curCover.IsNew = true
 	return nil
 }
 
-func (s *Scanner) handleFolder(it *item) error {
+func logElapsed(start time.Time, name string) {
-	// TODO:
+	elapsed := time.Since(start)
-	var folder model.Folder
+	log.Printf("finished %s in %s\n", name, elapsed)
-	err := s.tx.
-		Where("path = ?", it.relPath).
-		First(&folder).
-		Error
-	if !gorm.IsRecordNotFoundError(err) &&
-		it.stat.ModTime().Before(folder.UpdatedAt) {
-		// we found the record but it hasn't changed
-		s.curFolders.Push(folder)
-		return nil
-	}
-	folder.Path = it.relPath
-	folder.Name = it.stat.Name()
-	s.tx.Save(&folder)
-	folder.IsNew = true
-	s.curFolders.Push(folder)
-	return nil
-}
-
-func (s *Scanner) handleTrack(it *item) error {
-	//
-	// set track basics
-	track := model.Track{}
-	err := s.tx.
-		Where("path = ?", it.relPath).
-		First(&track).
-		Error
-	if !gorm.IsRecordNotFoundError(err) &&
-		it.stat.ModTime().Before(track.UpdatedAt) {
-		// we found the record but it hasn't changed
-		return nil
-	}
-	tags, err := readTags(it.path)
-	if err != nil {
-		return errors.Wrap(err, "reading tags")
-	}
-	trackNumber, totalTracks := tags.Track()
-	discNumber, totalDiscs := tags.Disc()
-	track.DiscNumber = discNumber
-	track.TotalDiscs = totalDiscs
-	track.TotalTracks = totalTracks
-	track.TrackNumber = trackNumber
-	track.Path = it.relPath
-	track.Suffix = it.track.ext
-	track.ContentType = it.track.mime
-	track.Size = int(it.stat.Size())
-	track.Title = tags.Title()
-	track.Artist = tags.Artist()
-	track.Year = tags.Year()
-	track.FolderID = s.curFolders.PeekID()
-	//
-	// set album artist basics
-	err = s.tx.Where("name = ?", tags.AlbumArtist()).
-		First(&s.curAArtist).
-		Error
-	if gorm.IsRecordNotFoundError(err) {
-		s.curAArtist.Name = tags.AlbumArtist()
-		s.tx.Save(&s.curAArtist)
-	}
-	track.AlbumArtistID = s.curAArtist.ID
-	//
-	// set album if this is the first track in the folder
-	if len(s.curTracks) > 0 {
-		s.curTracks = append(s.curTracks, track)
-		return nil
-	}
-	s.curTracks = append(s.curTracks, track)
-	//
-	directory, _ := path.Split(it.relPath)
-	err = s.tx.
-		Where("path = ?", directory).
-		First(&s.curAlbum).
-		Error
-	if !gorm.IsRecordNotFoundError(err) {
-		// we found the record
-		return nil
-	}
-	s.curAlbum.Path = directory
-	s.curAlbum.Title = tags.Album()
-	s.curAlbum.Year = tags.Year()
-	s.curAlbum.AlbumArtistID = s.curAArtist.ID
-	s.curAlbum.IsNew = true
-	return nil
-}
-
-func (s *Scanner) handleFolderCompletion(path string, info *godirwalk.Dirent) error {
-	// in general in this function - if a model is not nil, then it
-	// has at least been looked up. if it has a id of 0, then it is
-	// a new record and needs to be inserted
-	if s.curCover.IsNew {
-		s.tx.Save(&s.curCover)
-	}
-	if s.curAlbum.IsNew {
-		s.curAlbum.CoverID = s.curCover.ID
-		s.tx.Save(&s.curAlbum)
-	}
-	folder := s.curFolders.Pop()
-	if folder.IsNew {
-		folder.ParentID = s.curFolders.PeekID()
-		folder.CoverID = s.curCover.ID
-		folder.HasTracks = len(s.curTracks) > 1
-		s.tx.Save(&folder)
-	}
-	for _, t := range s.curTracks {
-		t.FolderID = folder.ID
-		t.AlbumID = s.curAlbum.ID
-		s.tx.Save(&t)
-	}
-	//
-	s.curTracks = make([]model.Track, 0)
-	s.curCover = model.Cover{}
-	s.curAlbum = model.Album{}
-	s.curAArtist = model.AlbumArtist{}
-	//
-	log.Printf("processed folder `%s`\n", path)
-	return nil
-}
-
-func (s *Scanner) handleItem(path string, info *godirwalk.Dirent) error {
-	stat, err := os.Stat(path)
-	if err != nil {
-		return errors.Wrap(err, "stating")
-	}
-	relPath, err := filepath.Rel(s.musicPath, path)
-	if err != nil {
-		return errors.Wrap(err, "getting relative path")
-	}
-	it := &item{
-		path:    path,
-		relPath: relPath,
-		stat:    stat,
-	}
-	if info.IsDir() {
-		return s.handleFolder(it)
-	}
-	if isCover(path) {
-		return s.handleCover(it)
-	}
-	if mime, ext, ok := isTrack(path); ok {
-		s.seenTracks[relPath] = true
-		it.track = &trackItem{mime: mime, ext: ext}
-		return s.handleTrack(it)
-	}
-	return nil
 }
 
 func (s *Scanner) startScan() error {
 	defer logElapsed(time.Now(), "scanning")
 	err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
-		Callback:             s.handleItem,
+		Callback:             s.callbackItem,
-		PostChildrenCallback: s.handleFolderCompletion,
+		PostChildrenCallback: s.callbackPost,
 		Unsorted:             true,
 	})
 	if err != nil {
@@ -269,7 +101,6 @@ func (s *Scanner) startClean() error {
 
 func (s *Scanner) MigrateDB() error {
 	defer logElapsed(time.Now(), "migrating database")
-	s.db.Exec("PRAGMA foreign_keys = ON")
 	s.tx = s.db.Begin()
 	defer s.tx.Commit()
 	s.tx.AutoMigrate(
@@ -289,21 +120,3 @@ func (s *Scanner) MigrateDB() error {
 	})
 	return nil
 }
-
-func (s *Scanner) Start() error {
-	if atomic.LoadInt32(&IsScanning) == 1 {
-		return errors.New("already scanning")
-	}
-	atomic.StoreInt32(&IsScanning, 1)
-	defer atomic.StoreInt32(&IsScanning, 0)
-	s.db.Exec("PRAGMA foreign_keys = ON")
-	s.tx = s.db.Begin()
-	defer s.tx.Commit()
-	if err := s.startScan(); err != nil {
-		return errors.Wrap(err, "start scan")
-	}
-	if err := s.startClean(); err != nil {
-		return errors.Wrap(err, "start clean")
-	}
-	return nil
-}

scanner/utilities.go

@@ -2,12 +2,10 @@ package scanner
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/dhowden/tag"
 )
@@ -62,8 +60,3 @@ func readTags(path string) (tag.Metadata, error) {
 	}
 	return tags, nil
 }
-
-func logElapsed(start time.Time, name string) {
-	elapsed := time.Since(start)
-	log.Printf("finished %s in %s\n", name, elapsed)
-}

scanner/walk.go

@@ -0,0 +1,195 @@
+package scanner
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/jinzhu/gorm"
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+
+	"github.com/sentriz/gonic/model"
+)
+
+type trackItem struct {
+	mime string
+	ext  string
+}
+
+type item struct {
+	path    string
+	relPath string
+	stat    os.FileInfo
+	track   *trackItem
+}
+
+func (s *Scanner) callbackItem(path string, info *godirwalk.Dirent) error {
+	stat, err := os.Stat(path)
+	if err != nil {
+		return errors.Wrap(err, "stating")
+	}
+	relPath, err := filepath.Rel(s.musicPath, path)
+	if err != nil {
+		return errors.Wrap(err, "getting relative path")
+	}
+	it := &item{
+		path:    path,
+		relPath: relPath,
+		stat:    stat,
+	}
+	if info.IsDir() {
+		return s.handleFolder(it)
+	}
+	if isCover(path) {
+		return s.handleCover(it)
+	}
+	if mime, ext, ok := isTrack(path); ok {
+		s.seenTracks[relPath] = true
+		it.track = &trackItem{mime: mime, ext: ext}
+		return s.handleTrack(it)
+	}
+	return nil
+}
+
+func (s *Scanner) callbackPost(path string, info *godirwalk.Dirent) error {
+	// in general in this function - if a model is not nil, then it
+	// has at least been looked up. if it has a id of 0, then it is
+	// a new record and needs to be inserted
+	if s.curCover.IsNew {
+		s.tx.Save(&s.curCover)
+	}
+	if s.curAlbum.IsNew {
+		s.curAlbum.CoverID = s.curCover.ID
+		s.tx.Save(&s.curAlbum)
+	}
+	folder := s.curFolders.Pop()
+	if folder.IsNew {
+		folder.ParentID = s.curFolders.PeekID()
+		folder.CoverID = s.curCover.ID
+		folder.HasTracks = len(s.curTracks) > 1
+		s.tx.Save(&folder)
+	}
+	for _, t := range s.curTracks {
+		t.FolderID = folder.ID
+		t.AlbumID = s.curAlbum.ID
+		s.tx.Save(&t)
+	}
+	//
+	s.curTracks = make([]model.Track, 0)
+	s.curCover = model.Cover{}
+	s.curAlbum = model.Album{}
+	s.curAArtist = model.AlbumArtist{}
+	//
+	log.Printf("processed folder `%s`\n", path)
+	return nil
+}
+
+func (s *Scanner) handleFolder(it *item) error {
+	// TODO:
+	var folder model.Folder
+	err := s.tx.
+		Where("path = ?", it.relPath).
+		First(&folder).
+		Error
+	if !gorm.IsRecordNotFoundError(err) &&
+		it.stat.ModTime().Before(folder.UpdatedAt) {
+		// we found the record but it hasn't changed
+		s.curFolders.Push(folder)
+		return nil
+	}
+	folder.Path = it.relPath
+	folder.Name = it.stat.Name()
+	s.tx.Save(&folder)
+	folder.IsNew = true
+	s.curFolders.Push(folder)
+	return nil
+}
+
+func (s *Scanner) handleCover(it *item) error {
+	err := s.tx.
+		Where("path = ?", it.relPath).
+		First(&s.curCover).
+		Error
+	if !gorm.IsRecordNotFoundError(err) &&
+		it.stat.ModTime().Before(s.curCover.UpdatedAt) {
+		// we found the record but it hasn't changed
+		return nil
+	}
+	s.curCover.Path = it.relPath
+	image, err := ioutil.ReadFile(it.path)
+	if err != nil {
+		return errors.Wrap(err, "reading cover")
+	}
+	s.curCover.Image = image
+	s.curCover.IsNew = true
+	return nil
+}
+
+func (s *Scanner) handleTrack(it *item) error {
+	//
+	// set track basics
+	track := model.Track{}
+	err := s.tx.
+		Where("path = ?", it.relPath).
+		First(&track).
+		Error
+	if !gorm.IsRecordNotFoundError(err) &&
+		it.stat.ModTime().Before(track.UpdatedAt) {
+		// we found the record but it hasn't changed
+		return nil
+	}
+	tags, err := readTags(it.path)
+	if err != nil {
+		return errors.Wrap(err, "reading tags")
+	}
+	trackNumber, totalTracks := tags.Track()
+	discNumber, totalDiscs := tags.Disc()
+	track.DiscNumber = discNumber
+	track.TotalDiscs = totalDiscs
+	track.TotalTracks = totalTracks
+	track.TrackNumber = trackNumber
+	track.Path = it.relPath
+	track.Suffix = it.track.ext
+	track.ContentType = it.track.mime
+	track.Size = int(it.stat.Size())
+	track.Title = tags.Title()
+	track.Artist = tags.Artist()
+	track.Year = tags.Year()
+	track.FolderID = s.curFolders.PeekID()
+	//
+	// set album artist basics
+	err = s.tx.Where("name = ?", tags.AlbumArtist()).
+		First(&s.curAArtist).
+		Error
+	if gorm.IsRecordNotFoundError(err) {
+		s.curAArtist.Name = tags.AlbumArtist()
+		s.tx.Save(&s.curAArtist)
+	}
+	track.AlbumArtistID = s.curAArtist.ID
+	//
+	// set album if this is the first track in the folder
+	if len(s.curTracks) > 0 {
+		s.curTracks = append(s.curTracks, track)
+		return nil
+	}
+	s.curTracks = append(s.curTracks, track)
+	//
+	directory, _ := path.Split(it.relPath)
+	err = s.tx.
+		Where("path = ?", directory).
+		First(&s.curAlbum).
+		Error
+	if !gorm.IsRecordNotFoundError(err) {
+		// we found the record
+		return nil
+	}
+	s.curAlbum.Path = directory
+	s.curAlbum.Title = tags.Album()
+	s.curAlbum.Year = tags.Year()
+	s.curAlbum.AlbumArtistID = s.curAArtist.ID
+	s.curAlbum.IsNew = true
+	return nil
+}