refactor(scanner): follow symlinks, move context, update mockfs

related #174

server/mockfs/mockfs.go

@@ -3,9 +3,9 @@ package mockfs
 import (
 	"errors"
 	"fmt"
-	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -24,13 +24,8 @@ type MockFS struct {
 	db      *db.DB
 }
 
-func New(t testing.TB) *MockFS {
+func New(t testing.TB) *MockFS                        { return new(t, []string{""}) }
-	return new(t, []string{""})
+func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs) }
-}
-
-func NewWithDirs(t testing.TB, dirs []string) *MockFS {
-	return new(t, dirs)
-}
 
 func new(t testing.TB, dirs []string) *MockFS {
 	dbc, err := db.NewMock()
@@ -91,6 +86,11 @@ func (m *MockFS) CleanUp() {
 	}
 }
 
+func (m *MockFS) AddItems()                              { m.addItems("", false) }
+func (m *MockFS) AddItemsPrefix(prefix string)           { m.addItems(prefix, false) }
+func (m *MockFS) AddItemsWithCovers()                    { m.addItems("", true) }
+func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) }
+
 func (m *MockFS) addItems(prefix string, covers bool) {
 	p := func(format string, a ...interface{}) string {
 		return filepath.Join(prefix, fmt.Sprintf(format, a...))
@@ -113,11 +113,6 @@ func (m *MockFS) addItems(prefix string, covers bool) {
 	}
 }
 
-func (m *MockFS) AddItems()                              { m.addItems("", false) }
-func (m *MockFS) AddItemsPrefix(prefix string)           { m.addItems(prefix, false) }
-func (m *MockFS) AddItemsWithCovers()                    { m.addItems("", true) }
-func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) }
-
 func (m *MockFS) RemoveAll(path string) {
 	abspath := filepath.Join(m.dir, path)
 	if err := os.RemoveAll(abspath); err != nil {
@@ -125,20 +120,40 @@ func (m *MockFS) RemoveAll(path string) {
 	}
 }
 
+func (m *MockFS) Symlink(src, dest string) {
+	if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
+		m.t.Fatalf("mkdir: %v", err)
+	}
+	if err := os.Symlink(src, dest); err != nil {
+		m.t.Fatalf("symlink: %v", err)
+	}
+	src = filepath.Clean(src)
+	dest = filepath.Clean(dest)
+	for k, v := range m.reader.tags {
+		m.reader.tags[strings.Replace(k, src, dest, 1)] = v
+	}
+}
+
 func (m *MockFS) LogItems() {
 	m.t.Logf("\nitems")
-	var dirs int
+	var items int
-	err := filepath.Walk(m.dir, func(path string, info fs.FileInfo, err error) error {
+	err := filepath.WalkDir(m.dir, func(path string, info os.DirEntry, err error) error {
-		m.t.Logf("item %q", path)
+		if err != nil {
-		if info.IsDir() {
+			return err
-			dirs++
 		}
+		switch info.Type() {
+		case os.ModeSymlink:
+			m.t.Logf("item %q [sym]", path)
+		default:
+			m.t.Logf("item %q", path)
+		}
+		items++
 		return nil
 	})
 	if err != nil {
 		m.t.Fatalf("error logging items: %v", err)
 	}
-	m.t.Logf("total %d", dirs)
+	m.t.Logf("total %d", items)
 }
 
 func (m *MockFS) LogAlbums() {
@@ -179,6 +194,7 @@ func (m *MockFS) LogTracks() {
 		m.t.Logf("id %-3d aid %-3d filename %-10s tagtitle %-10s",
 			track.ID, track.AlbumID, track.Filename, track.TagTitle)
 	}
+	m.t.Logf("total %d", len(tracks))
 }
 
 func (m *MockFS) LogTrackGenres() {
@@ -191,6 +207,7 @@ func (m *MockFS) LogTrackGenres() {
 	for _, tg := range tgs {
 		m.t.Logf("tid %-3d gid %-3d", tg.TrackID, tg.GenreID)
 	}
+	m.t.Logf("total %d", len(tgs))
 }
 
 func (m *MockFS) AddTrack(path string) {

server/scanner/scanner.go

@@ -61,8 +61,9 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
 	defer atomic.StoreInt32(s.scanning, 0)
 
 	start := time.Now()
-	itemErrs := &multierr.Err{}
+	c := &ctx{
-	c := &collected{
+		full:       opts.IsFull,
+		errs:       &multierr.Err{},
 		seenTracks: map[int]struct{}{},
 		seenAlbums: map[int]struct{}{},
 	}
@@ -70,40 +71,15 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
 	log.Println("starting scan")
 	defer func() {
 		log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
-			durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len())
+			durSince(start), c.seenTracksNew, len(c.seenTracks), c.errs.Len())
 	}()
 
 	for _, dir := range s.musicDirs {
-		dirName := filepath.Base(dir)
 		err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
-			if err != nil {
+			return s.scanCallback(c, dir, absPath, d, err)
-				itemErrs.Add(err)
-				return nil
-			}
-			if !d.IsDir() {
-				return nil
-			}
-			if dir == absPath {
-				return nil
-			}
-
-			relPath, _ := filepath.Rel(dir, absPath)
-			log.Printf("processing folder `%s` `%s`", dirName, relPath)
-
-			tx := s.db.Begin()
-			if err := s.scanDir(tx, c, opts.IsFull, dir, relPath); err != nil {
-				itemErrs.Add(fmt.Errorf("%q: %w", absPath, err))
-				tx.Rollback()
-				return nil
-			}
-			if err := tx.Commit().Error; err != nil {
-				return fmt.Errorf("commit tx: %w", err)
-			}
-
-			return nil
 		})
 		if err != nil {
-			return fmt.Errorf("walk %q: %w", dir, err)
+			return fmt.Errorf("walk: %w", err)
 		}
 	}
 
@@ -124,15 +100,50 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
 		return fmt.Errorf("set scan time: %w", err)
 	}
 
-	if itemErrs.Len() > 0 {
+	if c.errs.Len() > 0 {
-		return itemErrs
+		return c.errs
 	}
 
 	return nil
 }
 
-func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, relPath string) error {
+func (s *Scanner) scanCallback(c *ctx, dir string, absPath string, d fs.DirEntry, err error) error {
-	absPath := filepath.Join(musicDir, relPath)
+	if err != nil {
+		c.errs.Add(err)
+		return nil
+	}
+	if dir == absPath {
+		return nil
+	}
+
+	switch d.Type() {
+	case os.ModeDir:
+	case os.ModeSymlink:
+		eval, _ := filepath.EvalSymlinks(absPath)
+		return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
+			subAbs = strings.Replace(subAbs, eval, absPath, 1)
+			return s.scanCallback(c, dir, subAbs, d, err)
+		})
+	default:
+		return nil
+	}
+
+	log.Printf("processing folder `%s`", absPath)
+
+	tx := s.db.Begin()
+	if err := s.scanDir(tx, c, dir, absPath); err != nil {
+		c.errs.Add(fmt.Errorf("%q: %w", absPath, err))
+		tx.Rollback()
+		return nil
+	}
+	if err := tx.Commit().Error; err != nil {
+		return fmt.Errorf("commit tx: %w", err)
+	}
+
+	return nil
+}
+
+func (s *Scanner) scanDir(tx *db.DB, c *ctx, musicDir string, absPath string) error {
 	items, err := os.ReadDir(absPath)
 	if err != nil {
 		return err
@@ -151,6 +162,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
 		}
 	}
 
+	relPath, _ := filepath.Rel(musicDir, absPath)
 	pdir, pbasename := filepath.Split(filepath.Dir(relPath))
 	parent := &db.Album{}
 	if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil {
@@ -173,16 +185,16 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
 
 	sort.Strings(tracks)
 	for i, basename := range tracks {
-		abspath := filepath.Join(musicDir, relPath, basename)
+		absPath := filepath.Join(musicDir, relPath, basename)
-		if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil {
+		if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, absPath); err != nil {
-			return fmt.Errorf("process %q: %w", "", err)
+			return fmt.Errorf("populate track %q: %w", basename, err)
 		}
 	}
 
 	return nil
 }
 
-func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, absPath string, isFull bool) error {
+func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *ctx, i int, album *db.Album, basename string, absPath string) error {
 	track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
 	if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
 		return fmt.Errorf("query track: %w", err)
@@ -194,7 +206,7 @@ func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, a
 	if err != nil {
 		return fmt.Errorf("stating %q: %w", basename, err)
 	}
-	if !isFull && stat.ModTime().Before(track.UpdatedAt) {
+	if !c.full && stat.ModTime().Before(track.UpdatedAt) {
 		return nil
 	}
 
@@ -480,7 +492,9 @@ func durSince(t time.Time) time.Duration {
 	return time.Since(t).Truncate(10 * time.Microsecond)
 }
 
-type collected struct {
+type ctx struct {
+	errs          *multierr.Err
+	full          bool
 	seenTracks    map[int]struct{}
 	seenAlbums    map[int]struct{}
 	seenTracksNew int

server/scanner/scanner_test.go

@@ -370,3 +370,74 @@ func TestMultiFolderWithSharedArtist(t *testing.T) {
 		is.True(album.Duration > 0)
 	}
 }
+
+func TestSymlinkedAlbum(t *testing.T) {
+	t.Parallel()
+	is := is.New(t)
+	m := mockfs.NewWithDirs(t, []string{"scan"})
+	defer m.CleanUp()
+
+	m.AddItemsPrefixWithCovers("temp")
+
+	tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-0", "album-0")
+	scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-sym", "album-0")
+	m.Symlink(tempAlbum0, scanAlbum0)
+
+	m.ScanAndClean()
+	m.LogTracks()
+	m.LogAlbums()
+
+	var track db.Track
+	is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
+	is.True(track.Album != nil)                                 // track has album
+	is.True(track.Album.Cover != "")                            // album has cover
+	is.Equal(track.Album.Parent.RightPath, "artist-sym")        // artist is sym
+
+	info, err := os.Stat(track.AbsPath())
+	is.NoErr(err)                     // track resolves
+	is.True(!info.IsDir())            // track resolves
+	is.True(!info.ModTime().IsZero()) // track resolves
+}
+
+func TestSymlinkedSubdiscs(t *testing.T) {
+	t.Parallel()
+	is := is.New(t)
+	m := mockfs.NewWithDirs(t, []string{"scan"})
+	defer m.CleanUp()
+
+	addItem := func(prefix, artist, album, disc, track string) {
+		p := fmt.Sprintf("%s/%s/%s/%s/%s", prefix, artist, album, disc, track)
+		m.AddTrack(p)
+		m.SetTags(p, func(tags *mockfs.Tags) {
+			tags.RawArtist = artist
+			tags.RawAlbumArtist = artist
+			tags.RawAlbum = album
+			tags.RawTitle = track
+		})
+	}
+
+	addItem("temp", "artist-a", "album-a", "disc-1", "track-1.flac")
+	addItem("temp", "artist-a", "album-a", "disc-1", "track-2.flac")
+	addItem("temp", "artist-a", "album-a", "disc-1", "track-3.flac")
+	addItem("temp", "artist-a", "album-a", "disc-2", "track-1.flac")
+	addItem("temp", "artist-a", "album-a", "disc-2", "track-2.flac")
+	addItem("temp", "artist-a", "album-a", "disc-2", "track-3.flac")
+
+	tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-a", "album-a")
+	scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-a", "album-sym")
+	m.Symlink(tempAlbum0, scanAlbum0)
+
+	m.ScanAndClean()
+	m.LogTracks()
+	m.LogAlbums()
+
+	var track db.Track
+	is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
+	is.True(track.Album != nil)                                 // track has album
+	is.Equal(track.Album.Parent.RightPath, "album-sym")         // artist is sym
+
+	info, err := os.Stat(track.AbsPath())
+	is.NoErr(err)                     // track resolves
+	is.True(!info.IsDir())            // track resolves
+	is.True(!info.ModTime().IsZero()) // track resolves
+}