From 9e12394acc6037c085fd10ff3b841d381a933bf2 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 13 Dec 2023 01:32:01 +0000 Subject: [PATCH] refactor(scanner): rename scan context to scan state --- mockfs/mockfs.go | 8 +-- scanner/scanner.go | 98 ++++++++++++++++++------------------ scanner/scanner_fuzz_test.go | 14 +++--- scanner/scanner_test.go | 16 +++--- 4 files changed, 68 insertions(+), 68 deletions(-) diff --git a/mockfs/mockfs.go b/mockfs/mockfs.go index 23eabc5..450c136 100644 --- a/mockfs/mockfs.go +++ b/mockfs/mockfs.go @@ -83,17 +83,17 @@ func (m *MockFS) DB() *db.DB { return m.db } func (m *MockFS) TmpDir() string { return m.dir } func (m *MockFS) TagReader() tagcommon.Reader { return m.tagReader } -func (m *MockFS) ScanAndClean() *scanner.Context { +func (m *MockFS) ScanAndClean() *scanner.State { m.t.Helper() - ctx, err := m.scanner.ScanAndClean(scanner.ScanOptions{}) + st, err := m.scanner.ScanAndClean(scanner.ScanOptions{}) if err != nil { m.t.Fatalf("error scan and cleaning: %v", err) } - return ctx + return st } -func (m *MockFS) ScanAndCleanErr() (*scanner.Context, error) { +func (m *MockFS) ScanAndCleanErr() (*scanner.State, error) { m.t.Helper() return m.scanner.ScanAndClean(scanner.ScanOptions{}) diff --git a/scanner/scanner.go b/scanner/scanner.go index 863379a..8e4aa98 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -61,14 +61,14 @@ type ScanOptions struct { IsFull bool } -func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { +func (s *Scanner) ScanAndClean(opts ScanOptions) (*State, error) { if !s.StartScanning() { return nil, ErrAlreadyScanning } defer s.StopScanning() start := time.Now() - c := &Context{ + st := &State{ seenTracks: map[int]struct{}{}, seenAlbums: map[int]struct{}{}, isFull: opts.IsFull, @@ -77,28 +77,28 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { log.Println("starting scan") defer func() { log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n", - durSince(start), c.SeenTracksNew(), c.SeenTracks(), len(c.errs)) + durSince(start), st.SeenTracksNew(), st.SeenTracks(), len(st.errs)) }() for _, dir := range s.musicDirs { err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error { - return s.scanCallback(c, absPath, d, err) + return s.scanCallback(st, absPath, d, err) }) if err != nil { return nil, fmt.Errorf("walk: %w", err) } } - if err := s.cleanTracks(c); err != nil { + if err := s.cleanTracks(st); err != nil { return nil, fmt.Errorf("clean tracks: %w", err) } - if err := s.cleanAlbums(c); err != nil { + if err := s.cleanAlbums(st); err != nil { return nil, fmt.Errorf("clean albums: %w", err) } - if err := s.cleanArtists(c); err != nil { + if err := s.cleanArtists(st); err != nil { return nil, fmt.Errorf("clean artists: %w", err) } - if err := s.cleanGenres(c); err != nil { + if err := s.cleanGenres(st); err != nil { return nil, fmt.Errorf("clean genres: %w", err) } @@ -106,7 +106,7 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { return nil, fmt.Errorf("set scan time: %w", err) } - return c, errors.Join(c.errs...) + return st, errors.Join(st.errs...) } func (s *Scanner) ExecuteWatch(done <-chan struct{}) error { @@ -138,7 +138,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error { break } for absPath := range batchSeen { - c := &Context{ + st := &State{ seenTracks: map[int]struct{}{}, seenAlbums: map[int]struct{}{}, } @@ -150,7 +150,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error { continue } err = filepath.WalkDir(absPath, func(absPath string, d fs.DirEntry, err error) error { - return s.scanCallback(c, absPath, d, err) + return s.scanCallback(st, absPath, d, err) }) if err != nil { log.Printf("error walking: %v", err) @@ -207,9 +207,9 @@ func watchCallback(watcher *fsnotify.Watcher, absPath string, d fs.DirEntry, err return nil } -func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err error) error { +func (s *Scanner) scanCallback(st *State, absPath string, d fs.DirEntry, err error) error { if err != nil { - c.errs = append(c.errs, err) + st.errs = append(st.errs, err) return nil } @@ -219,7 +219,7 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er eval, _ := filepath.EvalSymlinks(absPath) return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error { subAbs = strings.Replace(subAbs, eval, absPath, 1) - return s.scanCallback(c, subAbs, d, err) + return s.scanCallback(st, subAbs, d, err) }) default: return nil @@ -233,15 +233,15 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er log.Printf("processing folder %q", absPath) return s.db.Transaction(func(tx *db.DB) error { - if err := s.scanDir(tx, c, absPath); err != nil { - c.errs = append(c.errs, fmt.Errorf("%q: %w", absPath, err)) + if err := s.scanDir(tx, st, absPath); err != nil { + st.errs = append(st.errs, fmt.Errorf("%q: %w", absPath, err)) return nil } return nil }) } -func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error { +func (s *Scanner) scanDir(tx *db.DB, st *State, absPath string) error { musicDir, relPath := musicDirRelative(s.musicDirs, absPath) if musicDir == absPath { return nil @@ -280,7 +280,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error { return fmt.Errorf("first or create parent: %w", err) } - c.seenAlbums[parent.ID] = struct{}{} + st.seenAlbums[parent.ID] = struct{}{} dir, basename := filepath.Split(relPath) var album db.Album @@ -288,12 +288,12 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error { return fmt.Errorf("populate album basics: %w", err) } - c.seenAlbums[album.ID] = struct{}{} + st.seenAlbums[album.ID] = struct{}{} sort.Strings(tracks) for i, basename := range tracks { absPath := filepath.Join(musicDir, relPath, basename) - if err := s.populateTrackAndArtists(tx, c, i, &album, basename, absPath); err != nil { + if err := s.populateTrackAndArtists(tx, st, i, &album, basename, absPath); err != nil { return fmt.Errorf("populate track %q: %w", basename, err) } } @@ -301,7 +301,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error { return nil } -func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *db.Album, basename string, absPath string) error { +func (s *Scanner) populateTrackAndArtists(tx *db.DB, st *State, i int, album *db.Album, basename string, absPath string) error { stat, err := os.Stat(absPath) if err != nil { return fmt.Errorf("stating %q: %w", basename, err) @@ -312,8 +312,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d return fmt.Errorf("query track: %w", err) } - if !c.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) { - c.seenTracks[track.ID] = struct{}{} + if !st.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) { + st.seenTracks[track.ID] = struct{}{} return nil } @@ -384,8 +384,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d return fmt.Errorf("populate track artists: %w", err) } - c.seenTracks[track.ID] = struct{}{} - c.seenTracksNew++ + st.seenTracks[track.ID] = struct{}{} + st.seenTracksNew++ return nil } @@ -540,9 +540,9 @@ func populateArtistAppearances(tx *db.DB, album *db.Album, artistIDs []int) erro return nil } -func (s *Scanner) cleanTracks(c *Context) error { +func (s *Scanner) cleanTracks(st *State) error { start := time.Now() - defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), c.TracksMissing()) }() + defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), st.TracksMissing()) }() var all []int err := s.db. @@ -553,18 +553,18 @@ func (s *Scanner) cleanTracks(c *Context) error { return fmt.Errorf("plucking ids: %w", err) } for _, a := range all { - if _, ok := c.seenTracks[a]; !ok { - c.tracksMissing = append(c.tracksMissing, int64(a)) + if _, ok := st.seenTracks[a]; !ok { + st.tracksMissing = append(st.tracksMissing, int64(a)) } } - return s.db.TransactionChunked(c.tracksMissing, func(tx *db.DB, chunk []int64) error { + return s.db.TransactionChunked(st.tracksMissing, func(tx *db.DB, chunk []int64) error { return tx.Where(chunk).Delete(&db.Track{}).Error }) } -func (s *Scanner) cleanAlbums(c *Context) error { +func (s *Scanner) cleanAlbums(st *State) error { start := time.Now() - defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), c.AlbumsMissing()) }() + defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), st.AlbumsMissing()) }() var all []int err := s.db. @@ -575,18 +575,18 @@ func (s *Scanner) cleanAlbums(c *Context) error { return fmt.Errorf("plucking ids: %w", err) } for _, a := range all { - if _, ok := c.seenAlbums[a]; !ok { - c.albumsMissing = append(c.albumsMissing, int64(a)) + if _, ok := st.seenAlbums[a]; !ok { + st.albumsMissing = append(st.albumsMissing, int64(a)) } } - return s.db.TransactionChunked(c.albumsMissing, func(tx *db.DB, chunk []int64) error { + return s.db.TransactionChunked(st.albumsMissing, func(tx *db.DB, chunk []int64) error { return tx.Where(chunk).Delete(&db.Album{}).Error }) } -func (s *Scanner) cleanArtists(c *Context) error { +func (s *Scanner) cleanArtists(st *State) error { start := time.Now() - defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), c.ArtistsMissing()) }() + defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), st.ArtistsMissing()) }() // gorm doesn't seem to support subqueries without parens for UNION q := s.db.Exec(` @@ -602,13 +602,13 @@ func (s *Scanner) cleanArtists(c *Context) error { if err := q.Error; err != nil { return err } - c.artistsMissing = int(q.RowsAffected) + st.artistsMissing = int(q.RowsAffected) return nil } -func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam +func (s *Scanner) cleanGenres(st *State) error { //nolint:unparam start := time.Now() - defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), c.GenresMissing()) }() + defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), st.GenresMissing()) }() subTrack := s.db. Select("genres.id"). @@ -625,7 +625,7 @@ func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam q := s.db. Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum). Delete(&db.Genre{}) - c.genresMissing = int(q.RowsAffected) + st.genresMissing = int(q.RowsAffected) return nil } @@ -663,7 +663,7 @@ func durSince(t time.Time) time.Duration { return time.Since(t).Truncate(10 * time.Microsecond) } -type Context struct { +type State struct { errs []error isFull bool @@ -677,14 +677,14 @@ type Context struct { genresMissing int } -func (c *Context) SeenTracks() int { return len(c.seenTracks) } -func (c *Context) SeenAlbums() int { return len(c.seenAlbums) } -func (c *Context) SeenTracksNew() int { return c.seenTracksNew } +func (s *State) SeenTracks() int { return len(s.seenTracks) } +func (s *State) SeenAlbums() int { return len(s.seenAlbums) } +func (s *State) SeenTracksNew() int { return s.seenTracksNew } -func (c *Context) TracksMissing() int { return len(c.tracksMissing) } -func (c *Context) AlbumsMissing() int { return len(c.albumsMissing) } -func (c *Context) ArtistsMissing() int { return c.artistsMissing } -func (c *Context) GenresMissing() int { return c.genresMissing } +func (s *State) TracksMissing() int { return len(s.tracksMissing) } +func (s *State) AlbumsMissing() int { return len(s.albumsMissing) } +func (s *State) ArtistsMissing() int { return s.artistsMissing } +func (s *State) GenresMissing() int { return s.genresMissing } type MultiValueMode uint8 diff --git a/scanner/scanner_fuzz_test.go b/scanner/scanner_fuzz_test.go index daf8e02..b4fc28b 100644 --- a/scanner/scanner_fuzz_test.go +++ b/scanner/scanner_fuzz_test.go @@ -16,13 +16,13 @@ import ( func FuzzScanner(f *testing.F) { checkDelta := func(assert *assert.Assertions, m *mockfs.MockFS, expSeen, expNew int) { - ctx := m.ScanAndClean() - assert.Equal(ctx.SeenTracks(), expSeen) - assert.Equal(ctx.SeenTracksNew(), expNew) - assert.Equal(ctx.TracksMissing(), 0) - assert.Equal(ctx.AlbumsMissing(), 0) - assert.Equal(ctx.ArtistsMissing(), 0) - assert.Equal(ctx.GenresMissing(), 0) + st := m.ScanAndClean() + assert.Equal(st.SeenTracks(), expSeen) + assert.Equal(st.SeenTracksNew(), expNew) + assert.Equal(st.TracksMissing(), 0) + assert.Equal(st.AlbumsMissing(), 0) + assert.Equal(st.ArtistsMissing(), 0) + assert.Equal(st.GenresMissing(), 0) } f.Fuzz(func(t *testing.T, data []byte, seed int64) { diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index dfec6e5..0541184 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -524,22 +524,22 @@ func TestTagErrors(t *testing.T) { tags.Error = scanner.ErrReadingTags }) - ctx, err := m.ScanAndCleanErr() + st, err := m.ScanAndCleanErr() errs, ok := err.(interface{ Unwrap() []error }) assert.True(t, ok) assert.ErrorAs(t, err, &errs) - assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors - assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents - assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracksNew()) // we have all tracks bar 2 album contents + assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors + assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents + assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracksNew()) // we have all tracks bar 2 album contents - ctx, err = m.ScanAndCleanErr() + st, err = m.ScanAndCleanErr() errs, ok = err.(interface{ Unwrap() []error }) assert.True(t, ok) - assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors - assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents - assert.Equal(t, 0, ctx.SeenTracksNew()) // we have no new tracks + assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors + assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents + assert.Equal(t, 0, st.SeenTracksNew()) // we have no new tracks } // https://github.com/sentriz/gonic/issues/185#issuecomment-1050092128