eg
This commit is contained in:
sentriz
303
scanner/scanner.go
Normal file
303
scanner/scanner.go
Normal file
@@ -0,0 +1,303 @@
|
||||
package scanner
|
||||
|
||||
// this scanner tries to scan with a single unsorted walk of the music
|
||||
// directory - which means you can come across the cover of an album/folder
|
||||
// before the tracks (and therefore the album) which is an issue because
|
||||
// when inserting into the album table, we need a reference to the cover.
|
||||
// to solve this we're using godirwalk's PostChildrenCallback and some globals
|
||||
//
|
||||
// Album -> needs a CoverID
|
||||
// -> needs a FolderID (American Football)
|
||||
// Folder -> needs a CoverID
|
||||
// -> needs a ParentID
|
||||
// Track -> needs an AlbumID
|
||||
// -> needs a FolderID
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
"github.com/karrick/godirwalk"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/sentriz/gonic/model"
|
||||
)
|
||||
|
||||
var (
|
||||
IsScanning int32
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
db *gorm.DB
|
||||
tx *gorm.DB
|
||||
musicPath string
|
||||
// seenPaths is used to keep every path we've seen so that
|
||||
// we can remove old tracks, folders, and covers by path when we
|
||||
// are in the cleanDatabase stage
|
||||
seenPaths map[string]bool
|
||||
// currentDirStack is used for inserting to the folders (subsonic browse
|
||||
// by folder) which helps us work out a folder's parent
|
||||
currentDirStack dirStack
|
||||
// currentCover because we find a cover anywhere among the tracks during the
|
||||
// walk and need a reference to it when we update folder and album records
|
||||
// when we exit a folder
|
||||
currentCover *model.Cover
|
||||
// currentAlbum because we update this record when we exit a folder with
|
||||
// our new reference to it's cover
|
||||
currentAlbum *model.Album
|
||||
}
|
||||
|
||||
func New(db *gorm.DB, musicPath string) *Scanner {
|
||||
return &Scanner{
|
||||
db: db,
|
||||
musicPath: musicPath,
|
||||
seenPaths: make(map[string]bool),
|
||||
currentDirStack: make(dirStack, 0),
|
||||
currentCover: &model.Cover{},
|
||||
currentAlbum: &model.Album{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Scanner) updateAlbum(fullPath string, album *model.Album) {
|
||||
if s.currentAlbum.ID != 0 {
|
||||
return
|
||||
}
|
||||
directory, _ := path.Split(fullPath)
|
||||
// update album table (the currentAlbum record will be updated when
|
||||
// we exit this folder)
|
||||
err := s.tx.Where("path = ?", directory).First(s.currentAlbum).Error
|
||||
if !gorm.IsRecordNotFoundError(err) {
|
||||
// we found the record
|
||||
// TODO: think about mod time here
|
||||
return
|
||||
}
|
||||
s.currentAlbum = &model.Album{
|
||||
Path: directory,
|
||||
Title: album.Title,
|
||||
AlbumArtistID: album.AlbumArtistID,
|
||||
Year: album.Year,
|
||||
}
|
||||
s.tx.Save(s.currentAlbum)
|
||||
}
|
||||
|
||||
func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error {
|
||||
modTime := stat.ModTime()
|
||||
err := s.tx.Where("path = ?", fullPath).First(s.currentCover).Error
|
||||
if !gorm.IsRecordNotFoundError(err) &&
|
||||
modTime.Before(s.currentCover.UpdatedAt) {
|
||||
// we found the record but it hasn't changed
|
||||
return nil
|
||||
}
|
||||
image, err := ioutil.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("when reading cover: %v", err)
|
||||
}
|
||||
s.currentCover = &model.Cover{
|
||||
Path: fullPath,
|
||||
Image: image,
|
||||
NewlyInserted: true,
|
||||
}
|
||||
s.tx.Save(s.currentCover)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error {
|
||||
// update folder table for browsing by folder
|
||||
folder := &model.Folder{}
|
||||
defer s.currentDirStack.Push(folder)
|
||||
modTime := stat.ModTime()
|
||||
err := s.tx.Where("path = ?", fullPath).First(folder).Error
|
||||
if !gorm.IsRecordNotFoundError(err) &&
|
||||
modTime.Before(folder.UpdatedAt) {
|
||||
// we found the record but it hasn't changed
|
||||
return nil
|
||||
}
|
||||
_, folderName := path.Split(fullPath)
|
||||
folder.Path = fullPath
|
||||
folder.ParentID = s.currentDirStack.PeekID()
|
||||
folder.Name = folderName
|
||||
s.tx.Save(folder)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error {
|
||||
currentDir := s.currentDirStack.Peek()
|
||||
defer s.currentDirStack.Pop()
|
||||
var dirShouldSave bool
|
||||
if s.currentAlbum.ID != 0 {
|
||||
s.currentAlbum.CoverID = s.currentCover.ID
|
||||
s.tx.Save(s.currentAlbum)
|
||||
currentDir.HasTracks = true
|
||||
dirShouldSave = true
|
||||
}
|
||||
if s.currentCover.NewlyInserted {
|
||||
currentDir.CoverID = s.currentCover.ID
|
||||
dirShouldSave = true
|
||||
}
|
||||
if dirShouldSave {
|
||||
s.tx.Save(currentDir)
|
||||
}
|
||||
s.currentCover = &model.Cover{}
|
||||
s.currentAlbum = &model.Album{}
|
||||
log.Printf("processed folder `%s`\n", fullPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error {
|
||||
//
|
||||
// set track basics
|
||||
track := &model.Track{}
|
||||
modTime := stat.ModTime()
|
||||
err := s.tx.Where("path = ?", fullPath).First(track).Error
|
||||
if !gorm.IsRecordNotFoundError(err) &&
|
||||
modTime.Before(track.UpdatedAt) {
|
||||
// we found the record but it hasn't changed
|
||||
return nil
|
||||
}
|
||||
tags, err := readTags(fullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("when reading tags: %v", err)
|
||||
}
|
||||
trackNumber, totalTracks := tags.Track()
|
||||
discNumber, totalDiscs := tags.Disc()
|
||||
track.Path = fullPath
|
||||
track.Title = tags.Title()
|
||||
track.Artist = tags.Artist()
|
||||
track.DiscNumber = discNumber
|
||||
track.TotalDiscs = totalDiscs
|
||||
track.TotalTracks = totalTracks
|
||||
track.TrackNumber = trackNumber
|
||||
track.Year = tags.Year()
|
||||
track.Suffix = exten
|
||||
track.ContentType = mime
|
||||
track.Size = int(stat.Size())
|
||||
track.FolderID = s.currentDirStack.PeekID()
|
||||
//
|
||||
// set album artist basics
|
||||
albumArtist := &model.AlbumArtist{}
|
||||
err = s.tx.Where("name = ?", tags.AlbumArtist()).
|
||||
First(albumArtist).
|
||||
Error
|
||||
if gorm.IsRecordNotFoundError(err) {
|
||||
albumArtist.Name = tags.AlbumArtist()
|
||||
s.tx.Save(albumArtist)
|
||||
}
|
||||
track.AlbumArtistID = albumArtist.ID
|
||||
//
|
||||
// set temporary album's basics - will be updated with
|
||||
// cover after the tracks inserted when we exit the folder
|
||||
s.updateAlbum(fullPath, &model.Album{
|
||||
AlbumArtistID: albumArtist.ID,
|
||||
Title: tags.Album(),
|
||||
Year: tags.Year(),
|
||||
})
|
||||
//
|
||||
// update the track with our new album and finally save
|
||||
track.AlbumID = s.currentAlbum.ID
|
||||
s.tx.Save(track)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) handleItem(fullPath string, info *godirwalk.Dirent) error {
|
||||
s.seenPaths[fullPath] = true
|
||||
stat, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error stating: %v", err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return s.handleFolder(fullPath, stat)
|
||||
}
|
||||
if isCover(fullPath) {
|
||||
return s.handleCover(fullPath, stat)
|
||||
}
|
||||
if mime, exten, ok := isAudio(fullPath); ok {
|
||||
return s.handleTrack(fullPath, stat, mime, exten)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) MigrateDB() error {
|
||||
defer logElapsed(time.Now(), "migrating database")
|
||||
s.tx = s.db.Begin()
|
||||
defer s.tx.Commit()
|
||||
s.tx.AutoMigrate(
|
||||
&model.Album{},
|
||||
&model.AlbumArtist{},
|
||||
&model.Track{},
|
||||
&model.Cover{},
|
||||
&model.User{},
|
||||
&model.Setting{},
|
||||
&model.Play{},
|
||||
&model.Folder{},
|
||||
)
|
||||
// set starting value for `albums` table's
|
||||
// auto increment
|
||||
s.tx.Exec(`
|
||||
INSERT INTO sqlite_sequence(name, seq)
|
||||
SELECT 'albums', 500000
|
||||
WHERE NOT EXISTS (SELECT *
|
||||
FROM sqlite_sequence);
|
||||
`)
|
||||
// create the first user if there is none
|
||||
s.tx.FirstOrCreate(&model.User{}, model.User{
|
||||
Name: "admin",
|
||||
Password: "admin",
|
||||
IsAdmin: true,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) Start() error {
|
||||
if atomic.LoadInt32(&IsScanning) == 1 {
|
||||
return errors.New("already scanning")
|
||||
}
|
||||
atomic.StoreInt32(&IsScanning, 1)
|
||||
defer atomic.StoreInt32(&IsScanning, 0)
|
||||
defer logElapsed(time.Now(), "scanning")
|
||||
s.tx = s.db.Begin()
|
||||
defer s.tx.Commit()
|
||||
//
|
||||
// start scan logic
|
||||
err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
|
||||
Callback: s.handleItem,
|
||||
PostChildrenCallback: s.handleFolderCompletion,
|
||||
Unsorted: true,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "walking filesystem")
|
||||
}
|
||||
//
|
||||
// start cleaning logic
|
||||
log.Println("cleaning database")
|
||||
var tracks []*model.Track
|
||||
s.tx.Select("id, path").Find(&tracks)
|
||||
for _, track := range tracks {
|
||||
_, ok := s.seenPaths[track.Path]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
s.tx.Delete(&track)
|
||||
log.Println("removed", track.Path)
|
||||
}
|
||||
// delete albums without tracks
|
||||
s.tx.Exec(`
|
||||
DELETE FROM albums
|
||||
WHERE (SELECT count(id)
|
||||
FROM tracks
|
||||
WHERE album_id = albums.id) = 0;
|
||||
`)
|
||||
// delete artists without tracks
|
||||
s.tx.Exec(`
|
||||
DELETE FROM album_artists
|
||||
WHERE (SELECT count(id)
|
||||
FROM albums
|
||||
WHERE album_artist_id = album_artists.id) = 0;
|
||||
`)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user