heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor scanner

Browse Source
This commit is contained in:
sentriz
2019-05-23 13:48:16 +01:00
parent ca37f5c27e
commit f8c5c41509
3 changed files with 114 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
model/model.go
View File

@@ -58,9 +58,8 @@ type Track struct {
type Cover struct { type Cover struct {
IDBase IDBase
CrudBase CrudBase
Image []byte Image []byte
Path string `gorm:"not null;unique_index"` Path string `gorm:"not null;unique_index"`
NewlyInserted bool `gorm:"-"`
} }
// User represents the users table // User represents the users table

14
scanner/dir_stack.go → scanner/folder_stack.go
View File

@@ -1,16 +1,14 @@
package scanner package scanner
import ( import "github.com/sentriz/gonic/model"
"github.com/sentriz/gonic/model"
)
type dirStack []*model.Folder type folderStack []*model.Folder
func (s *dirStack) Push(v *model.Folder) { func (s *folderStack) Push(v *model.Folder) {
*s = append(*s, v) *s = append(*s, v)
} }
func (s *dirStack) Pop() *model.Folder { func (s *folderStack) Pop() *model.Folder {
l := len(*s) l := len(*s)
if l == 0 { if l == 0 {
return nil return nil
@@ -20,7 +18,7 @@ func (s *dirStack) Pop() *model.Folder {
return r return r
} }
func (s *dirStack) Peek() *model.Folder { func (s *folderStack) Peek() *model.Folder {
l := len(*s) l := len(*s)
if l == 0 { if l == 0 {
return nil return nil
@@ -28,7 +26,7 @@ func (s *dirStack) Peek() *model.Folder {
return (*s)[l-1] return (*s)[l-1]
} }
func (s *dirStack) PeekID() int { func (s *folderStack) PeekID() int {
l := len(*s) l := len(*s)
if l == 0 { if l == 0 {
return 0 return 0

226
scanner/scanner.go
View File

@@ -34,117 +34,93 @@ var (
) )
type Scanner struct { type Scanner struct {
db *gorm.DB db *gorm.DB
tx *gorm.DB tx *gorm.DB
musicPath string musicPath string
// seenPaths is used to keep every path we've seen so that seenPaths map[string]bool
// we can remove old tracks, folders, and covers by path when we folderCount uint
// are in the cleanDatabase stage curFolders folderStack
seenPaths map[string]bool curTracks []*model.Track
// currentDirStack is used for inserting to the folders (subsonic browse curCover *model.Cover
// by folder) which helps us work out a folder's parent curAlbum *model.Album
currentDirStack dirStack curAArtist *model.AlbumArtist
// currentCover because we find a cover anywhere among the tracks during the
// walk and need a reference to it when we update folder and album records
// when we exit a folder
currentCover *model.Cover
// currentAlbum because we update this record when we exit a folder with
// our new reference to it's cover
currentAlbum *model.Album
} }
func New(db *gorm.DB, musicPath string) *Scanner { func New(db *gorm.DB, musicPath string) *Scanner {
return &Scanner{ return &Scanner{
db: db, db: db,
musicPath: musicPath, musicPath: musicPath,
seenPaths: make(map[string]bool), seenPaths: make(map[string]bool),
currentDirStack: make(dirStack, 0), curFolders: make(folderStack, 0),
currentCover: &model.Cover{},
currentAlbum: &model.Album{},
} }
} }
func (s *Scanner) updateAlbum(fullPath string, album *model.Album) {
if s.currentAlbum.ID != 0 {
return
}
directory, _ := path.Split(fullPath)
// update album table (the currentAlbum record will be updated when
// we exit this folder)
err := s.tx.Where("path = ?", directory).First(s.currentAlbum).Error
if !gorm.IsRecordNotFoundError(err) {
// we found the record
// TODO: think about mod time here
return
}
s.currentAlbum = &model.Album{
Path: directory,
Title: album.Title,
AlbumArtistID: album.AlbumArtistID,
Year: album.Year,
}
s.tx.Save(s.currentAlbum)
}
func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error { func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error {
modTime := stat.ModTime() s.curCover = &model.Cover{}
err := s.tx.Where("path = ?", fullPath).First(s.currentCover).Error err := s.tx.Where("path = ?", fullPath).First(s.curCover).Error
if !gorm.IsRecordNotFoundError(err) && if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(s.currentCover.UpdatedAt) { stat.ModTime().Before(s.curCover.UpdatedAt) {
// we found the record but it hasn't changed // we found the record but it hasn't changed
return nil return nil
} }
s.curCover.Path = fullPath
image, err := ioutil.ReadFile(fullPath) image, err := ioutil.ReadFile(fullPath)
if err != nil { if err != nil {
return fmt.Errorf("when reading cover: %v", err) return errors.Wrap(err, "reading cover")
} }
s.currentCover = &model.Cover{ s.curCover.Image = image
Path: fullPath,
Image: image,
NewlyInserted: true,
}
s.tx.Save(s.currentCover)
return nil return nil
} }
func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error { func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error {
// update folder table for browsing by folder
folder := &model.Folder{} folder := &model.Folder{}
defer s.currentDirStack.Push(folder) defer s.curFolders.Push(folder)
modTime := stat.ModTime()
err := s.tx.Where("path = ?", fullPath).First(folder).Error err := s.tx.Where("path = ?", fullPath).First(folder).Error
if !gorm.IsRecordNotFoundError(err) && if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(folder.UpdatedAt) { stat.ModTime().Before(folder.UpdatedAt) {
// we found the record but it hasn't changed // we found the record but it hasn't changed
return nil return nil
} }
_, folderName := path.Split(fullPath)
folder.Path = fullPath folder.Path = fullPath
folder.ParentID = s.currentDirStack.PeekID() folder.Name = stat.Name()
folder.Name = folderName
s.tx.Save(folder) s.tx.Save(folder)
return nil return nil
} }
func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error { func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error {
currentDir := s.currentDirStack.Peek() // in general in this function - if a model is not nil, then it
defer s.currentDirStack.Pop() // has at least been looked up. if it has a id of 0, then it is
var dirShouldSave bool // a new record and needs to be inserted
if s.currentAlbum.ID != 0 { //
s.currentAlbum.CoverID = s.currentCover.ID var newCover bool
s.tx.Save(s.currentAlbum) if s.curCover != nil && s.curCover.ID == 0 {
currentDir.HasTracks = true s.tx.Save(s.curCover)
dirShouldSave = true newCover = true
} }
if s.currentCover.NewlyInserted { if s.curAlbum != nil {
currentDir.CoverID = s.currentCover.ID s.curAlbum.CoverID = s.curCover.ID
dirShouldSave = true if newCover || s.curAlbum.ID == 0 {
s.tx.Save(s.curAlbum)
}
} }
if dirShouldSave { folder := s.curFolders.Pop()
s.tx.Save(currentDir) if folder.ID == 0 || newCover {
folder.ParentID = s.curFolders.PeekID()
folder.CoverID = s.curCover.ID
folder.HasTracks = true
s.tx.Save(folder)
} }
s.currentCover = &model.Cover{} for _, track := range s.curTracks {
s.currentAlbum = &model.Album{} // not checking for a nil album here because if there are
// tracks, then we at least lookup up the album
track.AlbumID = s.curAlbum.ID
track.FolderID = folder.ID
s.tx.Save(track)
}
s.curTracks = nil
s.curCover = nil
s.curAlbum = nil
s.curAArtist = nil
log.Printf("processed folder `%s`\n", fullPath) log.Printf("processed folder `%s`\n", fullPath)
return nil return nil
} }
@@ -177,30 +153,40 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str
track.Suffix = exten track.Suffix = exten
track.ContentType = mime track.ContentType = mime
track.Size = int(stat.Size()) track.Size = int(stat.Size())
track.FolderID = s.currentDirStack.PeekID() track.FolderID = s.curFolders.PeekID()
// //
// set album artist basics // set album artist basics
albumArtist := &model.AlbumArtist{} s.curAArtist = &model.AlbumArtist{}
err = s.tx.Where("name = ?", tags.AlbumArtist()). err = s.tx.Where("name = ?", tags.AlbumArtist()).
First(albumArtist). First(s.curAArtist).
Error Error
if gorm.IsRecordNotFoundError(err) { if gorm.IsRecordNotFoundError(err) {
albumArtist.Name = tags.AlbumArtist() s.curAArtist.Name = tags.AlbumArtist()
s.tx.Save(albumArtist) s.tx.Save(s.curAArtist)
} }
track.AlbumArtistID = albumArtist.ID track.AlbumArtistID = s.curAArtist.ID
// //
// set temporary album's basics - will be updated with // set album if this is the first track in the folder
// cover after the tracks inserted when we exit the folder if len(s.curTracks) > 0 {
s.updateAlbum(fullPath, &model.Album{ s.curTracks = append(s.curTracks, track)
AlbumArtistID: albumArtist.ID, return nil
Title: tags.Album(), }
Year: tags.Year(), s.curTracks = append(s.curTracks, track)
})
// //
// update the track with our new album and finally save s.curAlbum = &model.Album{}
track.AlbumID = s.currentAlbum.ID directory, _ := path.Split(fullPath)
s.tx.Save(track) err = s.tx.
Where("path = ?", directory).
First(s.curAlbum).
Error
if !gorm.IsRecordNotFoundError(err) {
// we found the record
return nil
}
s.curAlbum.Path = directory
s.curAlbum.Title = tags.Album()
s.curAlbum.Year = tags.Year()
s.curAlbum.AlbumArtistID = s.curAArtist.ID
return nil return nil
} }
@@ -265,32 +251,32 @@ func (s *Scanner) Start() error {
if err != nil { if err != nil {
return errors.Wrap(err, "walking filesystem") return errors.Wrap(err, "walking filesystem")
} }
// ////
// start cleaning logic //// start cleaning logic
log.Println("cleaning database") //log.Println("cleaning database")
var tracks []*model.Track //var tracks []*model.Track
s.tx.Select("id, path").Find(&tracks) //s.tx.Select("id, path").Find(&tracks)
for _, track := range tracks { //for _, track := range tracks {
_, ok := s.seenPaths[track.Path] // _, ok := s.seenPaths[track.Path]
if ok { // if ok {
continue // continue
} // }
s.tx.Delete(&track) // s.tx.Delete(&track)
log.Println("removed", track.Path) // log.Println("removed", track.Path)
} //}
// delete albums without tracks //// delete albums without tracks
s.tx.Exec(` //s.tx.Exec(`
DELETE FROM albums //DELETE FROM albums
WHERE (SELECT count(id) //WHERE (SELECT count(id)
FROM tracks //FROM tracks
WHERE album_id = albums.id) = 0; //WHERE album_id = albums.id) = 0;
`) //`)
// delete artists without tracks //// delete artists without tracks
s.tx.Exec(` //s.tx.Exec(`
DELETE FROM album_artists //DELETE FROM album_artists
WHERE (SELECT count(id) //WHERE (SELECT count(id)
FROM albums //FROM albums
WHERE album_artist_id = album_artists.id) = 0; //WHERE album_artist_id = album_artists.id) = 0;
`) //`)
return nil return nil
} }