heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(scanner): follow symlinks, move context, update mockfs

Browse Source 
related #174
This commit is contained in:
sentriz
2021-11-30 00:45:09 +00:00
committed by Senan Kelly
parent 62ccf01a3a
commit d9e8bd0d65
3 changed files with 161 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
server/mockfs/mockfs.go
View File

@@ -3,9 +3,9 @@ package mockfs
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -24,13 +24,8 @@ type MockFS struct {
db *db.DB
}
func New(t testing.TB) *MockFS {
return new(t, []string{""})
}
func NewWithDirs(t testing.TB, dirs []string) *MockFS {
return new(t, dirs)
}
func New(t testing.TB) *MockFS { return new(t, []string{""}) }
func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs) }
func new(t testing.TB, dirs []string) *MockFS {
dbc, err := db.NewMock()
@@ -91,6 +86,11 @@ func (m *MockFS) CleanUp() {
}
}
func (m *MockFS) AddItems() { m.addItems("", false) }
func (m *MockFS) AddItemsPrefix(prefix string) { m.addItems(prefix, false) }
func (m *MockFS) AddItemsWithCovers() { m.addItems("", true) }
func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) }
func (m *MockFS) addItems(prefix string, covers bool) {
p := func(format string, a ...interface{}) string {
return filepath.Join(prefix, fmt.Sprintf(format, a...))
@@ -113,11 +113,6 @@ func (m *MockFS) addItems(prefix string, covers bool) {
}
}
func (m *MockFS) AddItems() { m.addItems("", false) }
func (m *MockFS) AddItemsPrefix(prefix string) { m.addItems(prefix, false) }
func (m *MockFS) AddItemsWithCovers() { m.addItems("", true) }
func (m *MockFS) AddItemsPrefixWithCovers(prefix string) { m.addItems(prefix, true) }
func (m *MockFS) RemoveAll(path string) {
abspath := filepath.Join(m.dir, path)
if err := os.RemoveAll(abspath); err != nil {
@@ -125,20 +120,40 @@ func (m *MockFS) RemoveAll(path string) {
}
}
func (m *MockFS) Symlink(src, dest string) {
if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil {
m.t.Fatalf("mkdir: %v", err)
}
if err := os.Symlink(src, dest); err != nil {
m.t.Fatalf("symlink: %v", err)
}
src = filepath.Clean(src)
dest = filepath.Clean(dest)
for k, v := range m.reader.tags {
m.reader.tags[strings.Replace(k, src, dest, 1)] = v
}
}
func (m *MockFS) LogItems() {
m.t.Logf("\nitems")
var dirs int
err := filepath.Walk(m.dir, func(path string, info fs.FileInfo, err error) error {
m.t.Logf("item %q", path)
if info.IsDir() {
dirs++
var items int
err := filepath.WalkDir(m.dir, func(path string, info os.DirEntry, err error) error {
if err != nil {
return err
}
switch info.Type() {
case os.ModeSymlink:
m.t.Logf("item %q [sym]", path)
default:
m.t.Logf("item %q", path)
}
items++
return nil
})
if err != nil {
m.t.Fatalf("error logging items: %v", err)
}
m.t.Logf("total %d", dirs)
m.t.Logf("total %d", items)
}
func (m *MockFS) LogAlbums() {
@@ -179,6 +194,7 @@ func (m *MockFS) LogTracks() {
m.t.Logf("id %-3d aid %-3d filename %-10s tagtitle %-10s",
track.ID, track.AlbumID, track.Filename, track.TagTitle)
}
m.t.Logf("total %d", len(tracks))
}
func (m *MockFS) LogTrackGenres() {
@@ -191,6 +207,7 @@ func (m *MockFS) LogTrackGenres() {
for _, tg := range tgs {
m.t.Logf("tid %-3d gid %-3d", tg.TrackID, tg.GenreID)
}
m.t.Logf("total %d", len(tgs))
}
func (m *MockFS) AddTrack(path string) {

94
server/scanner/scanner.go
View File

@@ -61,8 +61,9 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
defer atomic.StoreInt32(s.scanning, 0)
start := time.Now()
itemErrs := &multierr.Err{}
c := &collected{
c := &ctx{
full: opts.IsFull,
errs: &multierr.Err{},
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
}
@@ -70,40 +71,15 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len())
durSince(start), c.seenTracksNew, len(c.seenTracks), c.errs.Len())
}()
for _, dir := range s.musicDirs {
dirName := filepath.Base(dir)
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
if err != nil {
itemErrs.Add(err)
return nil
}
if !d.IsDir() {
return nil
}
if dir == absPath {
return nil
}
relPath, _ := filepath.Rel(dir, absPath)
log.Printf("processing folder `%s` `%s`", dirName, relPath)
tx := s.db.Begin()
if err := s.scanDir(tx, c, opts.IsFull, dir, relPath); err != nil {
itemErrs.Add(fmt.Errorf("%q: %w", absPath, err))
tx.Rollback()
return nil
}
if err := tx.Commit().Error; err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
return s.scanCallback(c, dir, absPath, d, err)
})
if err != nil {
return fmt.Errorf("walk %q: %w", dir, err)
return fmt.Errorf("walk: %w", err)
}
}
@@ -124,15 +100,50 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) error {
return fmt.Errorf("set scan time: %w", err)
}
if itemErrs.Len() > 0 {
return itemErrs
if c.errs.Len() > 0 {
return c.errs
}
return nil
}
func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string, relPath string) error {
absPath := filepath.Join(musicDir, relPath)
func (s *Scanner) scanCallback(c *ctx, dir string, absPath string, d fs.DirEntry, err error) error {
if err != nil {
c.errs.Add(err)
return nil
}
if dir == absPath {
return nil
}
switch d.Type() {
case os.ModeDir:
case os.ModeSymlink:
eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.scanCallback(c, dir, subAbs, d, err)
})
default:
return nil
}
log.Printf("processing folder `%s`", absPath)
tx := s.db.Begin()
if err := s.scanDir(tx, c, dir, absPath); err != nil {
c.errs.Add(fmt.Errorf("%q: %w", absPath, err))
tx.Rollback()
return nil
}
if err := tx.Commit().Error; err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
}
func (s *Scanner) scanDir(tx *db.DB, c *ctx, musicDir string, absPath string) error {
items, err := os.ReadDir(absPath)
if err != nil {
return err
@@ -151,6 +162,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
}
}
relPath, _ := filepath.Rel(musicDir, absPath)
pdir, pbasename := filepath.Split(filepath.Dir(relPath))
parent := &db.Album{}
if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil {
@@ -173,16 +185,16 @@ func (s *Scanner) scanDir(tx *db.DB, c *collected, isFull bool, musicDir string,
sort.Strings(tracks)
for i, basename := range tracks {
abspath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil {
return fmt.Errorf("process %q: %w", "", err)
absPath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, absPath); err != nil {
return fmt.Errorf("populate track %q: %w", basename, err)
}
}
return nil
}
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, absPath string, isFull bool) error {
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *ctx, i int, album *db.Album, basename string, absPath string) error {
track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("query track: %w", err)
@@ -194,7 +206,7 @@ func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, a
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
}
if !isFull && stat.ModTime().Before(track.UpdatedAt) {
if !c.full && stat.ModTime().Before(track.UpdatedAt) {
return nil
}
@@ -480,7 +492,9 @@ func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
type collected struct {
type ctx struct {
errs *multierr.Err
full bool
seenTracks map[int]struct{}
seenAlbums map[int]struct{}
seenTracksNew int

71
server/scanner/scanner_test.go
View File

@@ -370,3 +370,74 @@ func TestMultiFolderWithSharedArtist(t *testing.T) {
is.True(album.Duration > 0)
}
}
func TestSymlinkedAlbum(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
defer m.CleanUp()
m.AddItemsPrefixWithCovers("temp")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-0", "album-0")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-sym", "album-0")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.True(track.Album.Cover != "") // album has cover
is.Equal(track.Album.Parent.RightPath, "artist-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}
func TestSymlinkedSubdiscs(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
defer m.CleanUp()
addItem := func(prefix, artist, album, disc, track string) {
p := fmt.Sprintf("%s/%s/%s/%s/%s", prefix, artist, album, disc, track)
m.AddTrack(p)
m.SetTags(p, func(tags *mockfs.Tags) {
tags.RawArtist = artist
tags.RawAlbumArtist = artist
tags.RawAlbum = album
tags.RawTitle = track
})
}
addItem("temp", "artist-a", "album-a", "disc-1", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-3.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-3.flac")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-a", "album-a")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-a", "album-sym")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.Equal(track.Album.Parent.RightPath, "album-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}