heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(scanner): follow symlinks, move context, update mockfs

Browse Source 
related #174
This commit is contained in:
sentriz
2021-11-30 00:45:09 +00:00
committed by Senan Kelly
parent 62ccf01a3a
commit d9e8bd0d65
3 changed files with 161 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
server/scanner/scanner.go
View File

@@ -61,8 +61,9 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
defer atomic.StoreInt32(s.scanning, 0)
start := time.Now()
itemErrs := &multierr.Err{}
c := &collected{
c := &ctx{
full: opts.IsFull,
errs: &multierr.Err{},
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
}
@@ -70,40 +71,15 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len())
durSince(start), c.seenTracksNew, len(c.seenTracks), c.errs.Len())
}()
for _, dir := range s.musicDirs {
dirName := filepath.Base(dir)
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
if err != nil {
itemErrs.Add(err)
return nil
}
if !d.IsDir() {
return nil
}
if dir == absPath {
return nil
}
relPath, _ := filepath.Rel(dir, absPath)
log.Printf("processing folder `%s` `%s`", dirName, relPath)
tx := s.db.Begin()
if err := s.scanDir(tx, c, opts.IsFull, dir, relPath); err != nil {
itemErrs.Add(fmt.Errorf("%q: %w", absPath, err))
tx.Rollback()
return nil
}
if err := tx.Commit().Error; err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
return s.scanCallback(c, dir, absPath, d, err)
})
if err != nil {
return fmt.Errorf("walk %q: %w", dir, err)
return fmt.Errorf("walk: %w", err)
}
}
@@ -124,15 +100,50 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
return fmt.Errorf("set scan time: %w", err)
}
if itemErrs.Len() > 0 {
return itemErrs
if c.errs.Len() > 0 {
return c.errs
}
return nil
}
func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, relPath string) error {
absPath := filepath.Join(musicDir, relPath)
func (s *Scanner) scanCallback(c *ctx, dir string, absPath string, d fs.DirEntry, err error) error {
if err != nil {
c.errs.Add(err)
return nil
}
if dir == absPath {
return nil
}
switch d.Type() {
case os.ModeDir:
case os.ModeSymlink:
eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.scanCallback(c, dir, subAbs, d, err)
})
default:
return nil
}
log.Printf("processing folder `%s`", absPath)
tx := s.db.Begin()
if err := s.scanDir(tx, c, dir, absPath); err != nil {
c.errs.Add(fmt.Errorf("%q: %w", absPath, err))
tx.Rollback()
return nil
}
if err := tx.Commit().Error; err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
}
func (s *Scanner) scanDir(tx *db.DB, c *ctx, musicDir string, absPath string) error {
items, err := os.ReadDir(absPath)
if err != nil {
return err
@@ -151,6 +162,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
}
}
relPath, _ := filepath.Rel(musicDir, absPath)
pdir, pbasename := filepath.Split(filepath.Dir(relPath))
parent := &db.Album{}
if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil {
@@ -173,16 +185,16 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
sort.Strings(tracks)
for i, basename := range tracks {
abspath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil {
return fmt.Errorf("process %q: %w", "", err)
absPath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, absPath); err != nil {
return fmt.Errorf("populate track %q: %w", basename, err)
}
}
return nil
}
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, absPath string, isFull bool) error {
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *ctx, i int, album *db.Album, basename string, absPath string) error {
track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("query track: %w", err)
@@ -194,7 +206,7 @@ func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, a
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
}
if !isFull && stat.ModTime().Before(track.UpdatedAt) {
if !c.full && stat.ModTime().Before(track.UpdatedAt) {
return nil
}
@@ -480,7 +492,9 @@ func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
type collected struct {
type ctx struct {
errs *multierr.Err
full bool
seenTracks map[int]struct{}
seenAlbums map[int]struct{}
seenTracksNew int

71
server/scanner/scanner_test.go
View File

@@ -370,3 +370,74 @@ func TestMultiFolderWithSharedArtist(t *testing.T) {
is.True(album.Duration > 0)
}
}
func TestSymlinkedAlbum(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
defer m.CleanUp()
m.AddItemsPrefixWithCovers("temp")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-0", "album-0")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-sym", "album-0")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.True(track.Album.Cover != "") // album has cover
is.Equal(track.Album.Parent.RightPath, "artist-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}
func TestSymlinkedSubdiscs(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
defer m.CleanUp()
addItem := func(prefix, artist, album, disc, track string) {
p := fmt.Sprintf("%s/%s/%s/%s/%s", prefix, artist, album, disc, track)
m.AddTrack(p)
m.SetTags(p, func(tags *mockfs.Tags) {
tags.RawArtist = artist
tags.RawAlbumArtist = artist
tags.RawAlbum = album
tags.RawTitle = track
})
}
addItem("temp", "artist-a", "album-a", "disc-1", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-3.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-3.flac")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-a", "album-a")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-a", "album-sym")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.Equal(track.Album.Parent.RightPath, "album-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}