diff --git a/server/mockfs/mockfs.go b/server/mockfs/mockfs.go index 33ebf09..7e6f036 100644 --- a/server/mockfs/mockfs.go +++ b/server/mockfs/mockfs.go @@ -3,9 +3,9 @@ package mockfs import ( "errors" "fmt" - "io/fs" "os" "path/filepath" + "strings" "testing" "time" @@ -24,13 +24,8 @@ type MockFS struct { db *db.DB } -func New(t testing.TB) *MockFS { - return new(t, []string{""}) -} - -func NewWithDirs(t testing.TB, dirs []string) *MockFS { - return new(t, dirs) -} +func New(t testing.TB) *MockFS { return new(t, []string{""}) } +func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs) } func new(t testing.TB, dirs []string) *MockFS { dbc, err := db.NewMock() @@ -91,6 +86,11 @@ func (m *MockFS) CleanUp() { } } +func (m *MockFS) AddItems() { m.addItems("", false) } +func (m *MockFS) AddItemsPrefix(prefix string) { m.addItems(prefix, false) } +func (m *MockFS) AddItemsWithCovers() { m.addItems("", true) } +func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) } + func (m *MockFS) addItems(prefix string, covers bool) { p := func(format string, a ...interface{}) string { return filepath.Join(prefix, fmt.Sprintf(format, a...)) @@ -113,11 +113,6 @@ func (m *MockFS) addItems(prefix string, covers bool) { } } -func (m *MockFS) AddItems() { m.addItems("", false) } -func (m *MockFS) AddItemsPrefix(prefix string) { m.addItems(prefix, false) } -func (m *MockFS) AddItemsWithCovers() { m.addItems("", true) } -func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) } - func (m *MockFS) RemoveAll(path string) { abspath := filepath.Join(m.dir, path) if err := os.RemoveAll(abspath); err != nil { @@ -125,20 +120,40 @@ func (m *MockFS) RemoveAll(path string) { } } +func (m *MockFS) Symlink(src, dest string) { + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + m.t.Fatalf("mkdir: %v", err) + } + if err := os.Symlink(src, dest); err != nil { + m.t.Fatalf("symlink: %v", err) + } + src = filepath.Clean(src) + dest = filepath.Clean(dest) + for k, v := range m.reader.tags { + m.reader.tags[strings.Replace(k, src, dest, 1)] = v + } +} + func (m *MockFS) LogItems() { m.t.Logf("\nitems") - var dirs int - err := filepath.Walk(m.dir, func(path string, info fs.FileInfo, err error) error { - m.t.Logf("item %q", path) - if info.IsDir() { - dirs++ + var items int + err := filepath.WalkDir(m.dir, func(path string, info os.DirEntry, err error) error { + if err != nil { + return err } + switch info.Type() { + case os.ModeSymlink: + m.t.Logf("item %q [sym]", path) + default: + m.t.Logf("item %q", path) + } + items++ return nil }) if err != nil { m.t.Fatalf("error logging items: %v", err) } - m.t.Logf("total %d", dirs) + m.t.Logf("total %d", items) } func (m *MockFS) LogAlbums() { @@ -179,6 +194,7 @@ func (m *MockFS) LogTracks() { m.t.Logf("id %-3d aid %-3d filename %-10s tagtitle %-10s", track.ID, track.AlbumID, track.Filename, track.TagTitle) } + m.t.Logf("total %d", len(tracks)) } func (m *MockFS) LogTrackGenres() { @@ -191,6 +207,7 @@ func (m *MockFS) LogTrackGenres() { for _, tg := range tgs { m.t.Logf("tid %-3d gid %-3d", tg.TrackID, tg.GenreID) } + m.t.Logf("total %d", len(tgs)) } func (m *MockFS) AddTrack(path string) { diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 062eaf5..18caa4d 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -61,8 +61,9 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error { defer atomic.StoreInt32(s.scanning, 0) start := time.Now() - itemErrs := &multierr.Err{} - c := &collected{ + c := &ctx{ + full: opts.IsFull, + errs: &multierr.Err{}, seenTracks: map[int]struct{}{}, seenAlbums: map[int]struct{}{}, } @@ -70,40 +71,15 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error { log.Println("starting scan") defer func() { log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n", - durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len()) + durSince(start), c.seenTracksNew, len(c.seenTracks), c.errs.Len()) }() for _, dir := range s.musicDirs { - dirName := filepath.Base(dir) err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error { - if err != nil { - itemErrs.Add(err) - return nil - } - if !d.IsDir() { - return nil - } - if dir == absPath { - return nil - } - - relPath, _ := filepath.Rel(dir, absPath) - log.Printf("processing folder `%s` `%s`", dirName, relPath) - - tx := s.db.Begin() - if err := s.scanDir(tx, c, opts.IsFull, dir, relPath); err != nil { - itemErrs.Add(fmt.Errorf("%q: %w", absPath, err)) - tx.Rollback() - return nil - } - if err := tx.Commit().Error; err != nil { - return fmt.Errorf("commit tx: %w", err) - } - - return nil + return s.scanCallback(c, dir, absPath, d, err) }) if err != nil { - return fmt.Errorf("walk %q: %w", dir, err) + return fmt.Errorf("walk: %w", err) } } @@ -124,15 +100,50 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error { return fmt.Errorf("set scan time: %w", err) } - if itemErrs.Len() > 0 { - return itemErrs + if c.errs.Len() > 0 { + return c.errs } return nil } -func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, relPath string) error { - absPath := filepath.Join(musicDir, relPath) +func (s *Scanner) scanCallback(c *ctx, dir string, absPath string, d fs.DirEntry, err error) error { + if err != nil { + c.errs.Add(err) + return nil + } + if dir == absPath { + return nil + } + + switch d.Type() { + case os.ModeDir: + case os.ModeSymlink: + eval, _ := filepath.EvalSymlinks(absPath) + return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error { + subAbs = strings.Replace(subAbs, eval, absPath, 1) + return s.scanCallback(c, dir, subAbs, d, err) + }) + default: + return nil + } + + log.Printf("processing folder `%s`", absPath) + + tx := s.db.Begin() + if err := s.scanDir(tx, c, dir, absPath); err != nil { + c.errs.Add(fmt.Errorf("%q: %w", absPath, err)) + tx.Rollback() + return nil + } + if err := tx.Commit().Error; err != nil { + return fmt.Errorf("commit tx: %w", err) + } + + return nil +} + +func (s *Scanner) scanDir(tx *db.DB, c *ctx, musicDir string, absPath string) error { items, err := os.ReadDir(absPath) if err != nil { return err @@ -151,6 +162,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, } } + relPath, _ := filepath.Rel(musicDir, absPath) pdir, pbasename := filepath.Split(filepath.Dir(relPath)) parent := &db.Album{} if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil { @@ -173,16 +185,16 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, sort.Strings(tracks) for i, basename := range tracks { - abspath := filepath.Join(musicDir, relPath, basename) - if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil { - return fmt.Errorf("process %q: %w", "", err) + absPath := filepath.Join(musicDir, relPath, basename) + if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, absPath); err != nil { + return fmt.Errorf("populate track %q: %w", basename, err) } } return nil } -func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, absPath string, isFull bool) error { +func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *ctx, i int, album *db.Album, basename string, absPath string) error { track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)} if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return fmt.Errorf("query track: %w", err) @@ -194,7 +206,7 @@ func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, a if err != nil { return fmt.Errorf("stating %q: %w", basename, err) } - if !isFull && stat.ModTime().Before(track.UpdatedAt) { + if !c.full && stat.ModTime().Before(track.UpdatedAt) { return nil } @@ -480,7 +492,9 @@ func durSince(t time.Time) time.Duration { return time.Since(t).Truncate(10 * time.Microsecond) } -type collected struct { +type ctx struct { + errs *multierr.Err + full bool seenTracks map[int]struct{} seenAlbums map[int]struct{} seenTracksNew int diff --git a/server/scanner/scanner_test.go b/server/scanner/scanner_test.go index a255746..4a6b5ba 100644 --- a/server/scanner/scanner_test.go +++ b/server/scanner/scanner_test.go @@ -370,3 +370,74 @@ func TestMultiFolderWithSharedArtist(t *testing.T) { is.True(album.Duration > 0) } } + +func TestSymlinkedAlbum(t *testing.T) { + t.Parallel() + is := is.New(t) + m := mockfs.NewWithDirs(t, []string{"scan"}) + defer m.CleanUp() + + m.AddItemsPrefixWithCovers("temp") + + tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-0", "album-0") + scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-sym", "album-0") + m.Symlink(tempAlbum0, scanAlbum0) + + m.ScanAndClean() + m.LogTracks() + m.LogAlbums() + + var track db.Track + is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists + is.True(track.Album != nil) // track has album + is.True(track.Album.Cover != "") // album has cover + is.Equal(track.Album.Parent.RightPath, "artist-sym") // artist is sym + + info, err := os.Stat(track.AbsPath()) + is.NoErr(err) // track resolves + is.True(!info.IsDir()) // track resolves + is.True(!info.ModTime().IsZero()) // track resolves +} + +func TestSymlinkedSubdiscs(t *testing.T) { + t.Parallel() + is := is.New(t) + m := mockfs.NewWithDirs(t, []string{"scan"}) + defer m.CleanUp() + + addItem := func(prefix, artist, album, disc, track string) { + p := fmt.Sprintf("%s/%s/%s/%s/%s", prefix, artist, album, disc, track) + m.AddTrack(p) + m.SetTags(p, func(tags *mockfs.Tags) { + tags.RawArtist = artist + tags.RawAlbumArtist = artist + tags.RawAlbum = album + tags.RawTitle = track + }) + } + + addItem("temp", "artist-a", "album-a", "disc-1", "track-1.flac") + addItem("temp", "artist-a", "album-a", "disc-1", "track-2.flac") + addItem("temp", "artist-a", "album-a", "disc-1", "track-3.flac") + addItem("temp", "artist-a", "album-a", "disc-2", "track-1.flac") + addItem("temp", "artist-a", "album-a", "disc-2", "track-2.flac") + addItem("temp", "artist-a", "album-a", "disc-2", "track-3.flac") + + tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-a", "album-a") + scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-a", "album-sym") + m.Symlink(tempAlbum0, scanAlbum0) + + m.ScanAndClean() + m.LogTracks() + m.LogAlbums() + + var track db.Track + is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists + is.True(track.Album != nil) // track has album + is.Equal(track.Album.Parent.RightPath, "album-sym") // artist is sym + + info, err := os.Stat(track.AbsPath()) + is.NoErr(err) // track resolves + is.True(!info.IsDir()) // track resolves + is.True(!info.ModTime().IsZero()) // track resolves +}