gonic/scanner/scanner.go

package scanner

// this scanner tries to scan with a single unsorted walk of the music
// directory - which means you can come across the cover of an album/folder
// before the tracks (and therefore the album) which is an issue because
// when inserting into the album table, we need a reference to the cover.
// to solve this we're using godirwalk's PostChildrenCallback and some globals
//
// Album  -> needs a  CoverID
//        -> needs a  FolderID (American Football)
// Folder -> needs a  CoverID
//        -> needs a  ParentID
// Track  -> needs an AlbumID
//        -> needs a  FolderID

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync/atomic"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/karrick/godirwalk"
	"github.com/pkg/errors"

	"github.com/sentriz/gonic/model"
)

var (
	IsScanning int32
)

type Scanner struct {
	db        *gorm.DB
	tx        *gorm.DB
	musicPath string
	// seenPaths is used to keep every path we've seen so that
	// we can remove old tracks, folders, and covers by path when we
	// are in the cleanDatabase stage
	seenPaths map[string]bool
	// currentDirStack is used for inserting to the folders (subsonic browse
	// by folder) which helps us work out a folder's parent
	currentDirStack dirStack
	// currentCover because we find a cover anywhere among the tracks during the
	// walk and need a reference to it when we update folder and album records
	// when we exit a folder
	currentCover *model.Cover
	// currentAlbum because we update this record when we exit a folder with
	// our new reference to it's cover
	currentAlbum *model.Album
}

func New(db *gorm.DB, musicPath string) *Scanner {
	return &Scanner{
		db:              db,
		musicPath:       musicPath,
		seenPaths:       make(map[string]bool),
		currentDirStack: make(dirStack, 0),
		currentCover:    &model.Cover{},
		currentAlbum:    &model.Album{},
	}
}

func (s *Scanner) updateAlbum(fullPath string, album *model.Album) {
	if s.currentAlbum.ID != 0 {
		return
	}
	directory, _ := path.Split(fullPath)
	// update album table (the currentAlbum record will be updated when
	// we exit this folder)
	err := s.tx.Where("path = ?", directory).First(s.currentAlbum).Error
	if !gorm.IsRecordNotFoundError(err) {
		// we found the record
		// TODO: think about mod time here
		return
	}
	s.currentAlbum = &model.Album{
		Path:          directory,
		Title:         album.Title,
		AlbumArtistID: album.AlbumArtistID,
		Year:          album.Year,
	}
	s.tx.Save(s.currentAlbum)
}

func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error {
	modTime := stat.ModTime()
	err := s.tx.Where("path = ?", fullPath).First(s.currentCover).Error
	if !gorm.IsRecordNotFoundError(err) &&
		modTime.Before(s.currentCover.UpdatedAt) {
		// we found the record but it hasn't changed
		return nil
	}
	image, err := ioutil.ReadFile(fullPath)
	if err != nil {
		return fmt.Errorf("when reading cover: %v", err)
	}
	s.currentCover = &model.Cover{
		Path:          fullPath,
		Image:         image,
		NewlyInserted: true,
	}
	s.tx.Save(s.currentCover)
	return nil
}

func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error {
	// update folder table for browsing by folder
	folder := &model.Folder{}
	defer s.currentDirStack.Push(folder)
	modTime := stat.ModTime()
	err := s.tx.Where("path = ?", fullPath).First(folder).Error
	if !gorm.IsRecordNotFoundError(err) &&
		modTime.Before(folder.UpdatedAt) {
		// we found the record but it hasn't changed
		return nil
	}
	_, folderName := path.Split(fullPath)
	folder.Path = fullPath
	folder.ParentID = s.currentDirStack.PeekID()
	folder.Name = folderName
	s.tx.Save(folder)
	return nil
}

func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error {
	currentDir := s.currentDirStack.Peek()
	defer s.currentDirStack.Pop()
	var dirShouldSave bool
	if s.currentAlbum.ID != 0 {
		s.currentAlbum.CoverID = s.currentCover.ID
		s.tx.Save(s.currentAlbum)
		currentDir.HasTracks = true
		dirShouldSave = true
	}
	if s.currentCover.NewlyInserted {
		currentDir.CoverID = s.currentCover.ID
		dirShouldSave = true
	}
	if dirShouldSave {
		s.tx.Save(currentDir)
	}
	s.currentCover = &model.Cover{}
	s.currentAlbum = &model.Album{}
	log.Printf("processed folder `%s`\n", fullPath)
	return nil
}

func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error {
	//
	// set track basics
	track := &model.Track{}
	modTime := stat.ModTime()
	err := s.tx.Where("path = ?", fullPath).First(track).Error
	if !gorm.IsRecordNotFoundError(err) &&
		modTime.Before(track.UpdatedAt) {
		// we found the record but it hasn't changed
		return nil
	}
	tags, err := readTags(fullPath)
	if err != nil {
		return fmt.Errorf("when reading tags: %v", err)
	}
	trackNumber, totalTracks := tags.Track()
	discNumber, totalDiscs := tags.Disc()
	track.Path = fullPath
	track.Title = tags.Title()
	track.Artist = tags.Artist()
	track.DiscNumber = discNumber
	track.TotalDiscs = totalDiscs
	track.TotalTracks = totalTracks
	track.TrackNumber = trackNumber
	track.Year = tags.Year()
	track.Suffix = exten
	track.ContentType = mime
	track.Size = int(stat.Size())
	track.FolderID = s.currentDirStack.PeekID()
	//
	// set album artist basics
	albumArtist := &model.AlbumArtist{}
	err = s.tx.Where("name = ?", tags.AlbumArtist()).
		First(albumArtist).
		Error
	if gorm.IsRecordNotFoundError(err) {
		albumArtist.Name = tags.AlbumArtist()
		s.tx.Save(albumArtist)
	}
	track.AlbumArtistID = albumArtist.ID
	//
	// set temporary album's basics - will be updated with
	// cover after the tracks inserted when we exit the folder
	s.updateAlbum(fullPath, &model.Album{
		AlbumArtistID: albumArtist.ID,
		Title:         tags.Album(),
		Year:          tags.Year(),
	})
	//
	// update the track with our new album and finally save
	track.AlbumID = s.currentAlbum.ID
	s.tx.Save(track)
	return nil
}

func (s *Scanner) handleItem(fullPath string, info *godirwalk.Dirent) error {
	s.seenPaths[fullPath] = true
	stat, err := os.Stat(fullPath)
	if err != nil {
		return fmt.Errorf("error stating: %v", err)
	}
	if info.IsDir() {
		return s.handleFolder(fullPath, stat)
	}
	if isCover(fullPath) {
		return s.handleCover(fullPath, stat)
	}
	if mime, exten, ok := isAudio(fullPath); ok {
		return s.handleTrack(fullPath, stat, mime, exten)
	}
	return nil
}

func (s *Scanner) MigrateDB() error {
	defer logElapsed(time.Now(), "migrating database")
	s.db.Exec("PRAGMA foreign_keys = ON")
	s.tx = s.db.Begin()
	defer s.tx.Commit()
	s.tx.AutoMigrate(
		model.Album{},
		model.AlbumArtist{},
		model.Track{},
		model.Cover{},
		model.User{},
		model.Setting{},
		model.Play{},
		model.Folder{},
	)
	s.tx.FirstOrCreate(&model.User{}, model.User{
		Name:     "admin",
		Password: "admin",
		IsAdmin:  true,
	})
	return nil
}

func (s *Scanner) Start() error {
	if atomic.LoadInt32(&IsScanning) == 1 {
		return errors.New("already scanning")
	}
	atomic.StoreInt32(&IsScanning, 1)
	defer atomic.StoreInt32(&IsScanning, 0)
	defer logElapsed(time.Now(), "scanning")
	s.db.Exec("PRAGMA foreign_keys = ON")
	s.tx = s.db.Begin()
	defer s.tx.Commit()
	//
	// start scan logic
	err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
		Callback:             s.handleItem,
		PostChildrenCallback: s.handleFolderCompletion,
		Unsorted:             true,
	})
	if err != nil {
		return errors.Wrap(err, "walking filesystem")
	}
	//
	// start cleaning logic
	log.Println("cleaning database")
	var tracks []*model.Track
	s.tx.Select("id, path").Find(&tracks)
	for _, track := range tracks {
		_, ok := s.seenPaths[track.Path]
		if ok {
			continue
		}
		s.tx.Delete(&track)
		log.Println("removed", track.Path)
	}
	// delete albums without tracks
	s.tx.Exec(`
        DELETE FROM albums
        WHERE  (SELECT count(id)
                FROM   tracks
                WHERE  album_id = albums.id) = 0;
       `)
	// delete artists without tracks
	s.tx.Exec(`
        DELETE FROM album_artists
        WHERE  (SELECT count(id)
                FROM   albums
                WHERE  album_artist_id = album_artists.id) = 0;
    `)
	return nil
}