heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(scanner): rename scan context to scan state

Browse Source
This commit is contained in:
sentriz
2023-12-13 01:32:01 +00:00
parent a892595641
commit 9e12394acc
4 changed files with 68 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
mockfs/mockfs.go
View File

@@ -83,17 +83,17 @@ func (m *MockFS) DB() *db.DB { return m.db }
func (m *MockFS) TmpDir() string { return m.dir } func (m *MockFS) TmpDir() string { return m.dir }
func (m *MockFS) TagReader() tagcommon.Reader { return m.tagReader } func (m *MockFS) TagReader() tagcommon.Reader { return m.tagReader }
func (m *MockFS) ScanAndClean() *scanner.Context { func (m *MockFS) ScanAndClean() *scanner.State {
m.t.Helper() m.t.Helper()
ctx, err := m.scanner.ScanAndClean(scanner.ScanOptions{}) st, err := m.scanner.ScanAndClean(scanner.ScanOptions{})
if err != nil { if err != nil {
m.t.Fatalf("error scan and cleaning: %v", err) m.t.Fatalf("error scan and cleaning: %v", err)
} }
return ctx return st
} }
func (m *MockFS) ScanAndCleanErr() (*scanner.Context, error) { func (m *MockFS) ScanAndCleanErr() (*scanner.State, error) {
m.t.Helper() m.t.Helper()
return m.scanner.ScanAndClean(scanner.ScanOptions{}) return m.scanner.ScanAndClean(scanner.ScanOptions{})

98
scanner/scanner.go
View File

@@ -61,14 +61,14 @@ type ScanOptions struct {
IsFull bool IsFull bool
} }
func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { func (s *Scanner) ScanAndClean(opts ScanOptions) (*State, error) {
if !s.StartScanning() { if !s.StartScanning() {
return nil, ErrAlreadyScanning return nil, ErrAlreadyScanning
} }
defer s.StopScanning() defer s.StopScanning()
start := time.Now() start := time.Now()
c := &Context{ st := &State{
seenTracks: map[int]struct{}{}, seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{}, seenAlbums: map[int]struct{}{},
isFull: opts.IsFull, isFull: opts.IsFull,
@@ -77,28 +77,28 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
log.Println("starting scan") log.Println("starting scan")
defer func() { defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n", log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.SeenTracksNew(), c.SeenTracks(), len(c.errs)) durSince(start), st.SeenTracksNew(), st.SeenTracks(), len(st.errs))
}() }()
for _, dir := range s.musicDirs { for _, dir := range s.musicDirs {
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error { err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, absPath, d, err) return s.scanCallback(st, absPath, d, err)
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("walk: %w", err) return nil, fmt.Errorf("walk: %w", err)
} }
} }
if err := s.cleanTracks(c); err != nil { if err := s.cleanTracks(st); err != nil {
return nil, fmt.Errorf("clean tracks: %w", err) return nil, fmt.Errorf("clean tracks: %w", err)
} }
if err := s.cleanAlbums(c); err != nil { if err := s.cleanAlbums(st); err != nil {
return nil, fmt.Errorf("clean albums: %w", err) return nil, fmt.Errorf("clean albums: %w", err)
} }
if err := s.cleanArtists(c); err != nil { if err := s.cleanArtists(st); err != nil {
return nil, fmt.Errorf("clean artists: %w", err) return nil, fmt.Errorf("clean artists: %w", err)
} }
if err := s.cleanGenres(c); err != nil { if err := s.cleanGenres(st); err != nil {
return nil, fmt.Errorf("clean genres: %w", err) return nil, fmt.Errorf("clean genres: %w", err)
} }
@@ -106,7 +106,7 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
return nil, fmt.Errorf("set scan time: %w", err) return nil, fmt.Errorf("set scan time: %w", err)
} }
return c, errors.Join(c.errs...) return st, errors.Join(st.errs...)
} }
func (s *Scanner) ExecuteWatch(done <-chan struct{}) error { func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
@@ -138,7 +138,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
break break
} }
for absPath := range batchSeen { for absPath := range batchSeen {
c := &Context{ st := &State{
seenTracks: map[int]struct{}{}, seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{}, seenAlbums: map[int]struct{}{},
} }
@@ -150,7 +150,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
continue continue
} }
err = filepath.WalkDir(absPath, func(absPath string, d fs.DirEntry, err error) error { err = filepath.WalkDir(absPath, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, absPath, d, err) return s.scanCallback(st, absPath, d, err)
}) })
if err != nil { if err != nil {
log.Printf("error walking: %v", err) log.Printf("error walking: %v", err)
@@ -207,9 +207,9 @@ func watchCallback(watcher *fsnotify.Watcher, absPath string, d fs.DirEntry, err
return nil return nil
} }
func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err error) error { func (s *Scanner) scanCallback(st *State, absPath string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
c.errs = append(c.errs, err) st.errs = append(st.errs, err)
return nil return nil
} }
@@ -219,7 +219,7 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er
eval, _ := filepath.EvalSymlinks(absPath) eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error { return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1) subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.scanCallback(c, subAbs, d, err) return s.scanCallback(st, subAbs, d, err)
}) })
default: default:
return nil return nil
@@ -233,15 +233,15 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er
log.Printf("processing folder %q", absPath) log.Printf("processing folder %q", absPath)
return s.db.Transaction(func(tx *db.DB) error { return s.db.Transaction(func(tx *db.DB) error {
if err := s.scanDir(tx, c, absPath); err != nil { if err := s.scanDir(tx, st, absPath); err != nil {
c.errs = append(c.errs, fmt.Errorf("%q: %w", absPath, err)) st.errs = append(st.errs, fmt.Errorf("%q: %w", absPath, err))
return nil return nil
} }
return nil return nil
}) })
} }
func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error { func (s *Scanner) scanDir(tx *db.DB, st *State, absPath string) error {
musicDir, relPath := musicDirRelative(s.musicDirs, absPath) musicDir, relPath := musicDirRelative(s.musicDirs, absPath)
if musicDir == absPath { if musicDir == absPath {
return nil return nil
@@ -280,7 +280,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return fmt.Errorf("first or create parent: %w", err) return fmt.Errorf("first or create parent: %w", err)
} }
c.seenAlbums[parent.ID] = struct{}{} st.seenAlbums[parent.ID] = struct{}{}
dir, basename := filepath.Split(relPath) dir, basename := filepath.Split(relPath)
var album db.Album var album db.Album
@@ -288,12 +288,12 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return fmt.Errorf("populate album basics: %w", err) return fmt.Errorf("populate album basics: %w", err)
} }
c.seenAlbums[album.ID] = struct{}{} st.seenAlbums[album.ID] = struct{}{}
sort.Strings(tracks) sort.Strings(tracks)
for i, basename := range tracks { for i, basename := range tracks {
absPath := filepath.Join(musicDir, relPath, basename) absPath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndArtists(tx, c, i, &album, basename, absPath); err != nil { if err := s.populateTrackAndArtists(tx, st, i, &album, basename, absPath); err != nil {
return fmt.Errorf("populate track %q: %w", basename, err) return fmt.Errorf("populate track %q: %w", basename, err)
} }
} }
@@ -301,7 +301,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return nil return nil
} }
func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *db.Album, basename string, absPath string) error { func (s *Scanner) populateTrackAndArtists(tx *db.DB, st *State, i int, album *db.Album, basename string, absPath string) error {
stat, err := os.Stat(absPath) stat, err := os.Stat(absPath)
if err != nil { if err != nil {
return fmt.Errorf("stating %q: %w", basename, err) return fmt.Errorf("stating %q: %w", basename, err)
@@ -312,8 +312,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d
return fmt.Errorf("query track: %w", err) return fmt.Errorf("query track: %w", err)
} }
if !c.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) { if !st.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) {
c.seenTracks[track.ID] = struct{}{} st.seenTracks[track.ID] = struct{}{}
return nil return nil
} }
@@ -384,8 +384,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d
return fmt.Errorf("populate track artists: %w", err) return fmt.Errorf("populate track artists: %w", err)
} }
c.seenTracks[track.ID] = struct{}{} st.seenTracks[track.ID] = struct{}{}
c.seenTracksNew++ st.seenTracksNew++
return nil return nil
} }
@@ -540,9 +540,9 @@ func populateArtistAppearances(tx *db.DB, album *db.Album, artistIDs []int) erro
return nil return nil
} }
func (s *Scanner) cleanTracks(c *Context) error { func (s *Scanner) cleanTracks(st *State) error {
start := time.Now() start := time.Now()
defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), c.TracksMissing()) }() defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), st.TracksMissing()) }()
var all []int var all []int
err := s.db. err := s.db.
@@ -553,18 +553,18 @@ func (s *Scanner) cleanTracks(c *Context) error {
return fmt.Errorf("plucking ids: %w", err) return fmt.Errorf("plucking ids: %w", err)
} }
for _, a := range all { for _, a := range all {
if _, ok := c.seenTracks[a]; !ok { if _, ok := st.seenTracks[a]; !ok {
c.tracksMissing = append(c.tracksMissing, int64(a)) st.tracksMissing = append(st.tracksMissing, int64(a))
} }
} }
return s.db.TransactionChunked(c.tracksMissing, func(tx *db.DB, chunk []int64) error { return s.db.TransactionChunked(st.tracksMissing, func(tx *db.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Track{}).Error return tx.Where(chunk).Delete(&db.Track{}).Error
}) })
} }
func (s *Scanner) cleanAlbums(c *Context) error { func (s *Scanner) cleanAlbums(st *State) error {
start := time.Now() start := time.Now()
defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), c.AlbumsMissing()) }() defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), st.AlbumsMissing()) }()
var all []int var all []int
err := s.db. err := s.db.
@@ -575,18 +575,18 @@ func (s *Scanner) cleanAlbums(c *Context) error {
return fmt.Errorf("plucking ids: %w", err) return fmt.Errorf("plucking ids: %w", err)
} }
for _, a := range all { for _, a := range all {
if _, ok := c.seenAlbums[a]; !ok { if _, ok := st.seenAlbums[a]; !ok {
c.albumsMissing = append(c.albumsMissing, int64(a)) st.albumsMissing = append(st.albumsMissing, int64(a))
} }
} }
return s.db.TransactionChunked(c.albumsMissing, func(tx *db.DB, chunk []int64) error { return s.db.TransactionChunked(st.albumsMissing, func(tx *db.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Album{}).Error return tx.Where(chunk).Delete(&db.Album{}).Error
}) })
} }
func (s *Scanner) cleanArtists(c *Context) error { func (s *Scanner) cleanArtists(st *State) error {
start := time.Now() start := time.Now()
defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), c.ArtistsMissing()) }() defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), st.ArtistsMissing()) }()
// gorm doesn't seem to support subqueries without parens for UNION // gorm doesn't seem to support subqueries without parens for UNION
q := s.db.Exec(` q := s.db.Exec(`
@@ -602,13 +602,13 @@ func (s *Scanner) cleanArtists(c *Context) error {
if err := q.Error; err != nil { if err := q.Error; err != nil {
return err return err
} }
c.artistsMissing = int(q.RowsAffected) st.artistsMissing = int(q.RowsAffected)
return nil return nil
} }
func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam func (s *Scanner) cleanGenres(st *State) error { //nolint:unparam
start := time.Now() start := time.Now()
defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), c.GenresMissing()) }() defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), st.GenresMissing()) }()
subTrack := s.db. subTrack := s.db.
Select("genres.id"). Select("genres.id").
@@ -625,7 +625,7 @@ func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam
q := s.db. q := s.db.
Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum). Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum).
Delete(&db.Genre{}) Delete(&db.Genre{})
c.genresMissing = int(q.RowsAffected) st.genresMissing = int(q.RowsAffected)
return nil return nil
} }
@@ -663,7 +663,7 @@ func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond) return time.Since(t).Truncate(10 * time.Microsecond)
} }
type Context struct { type State struct {
errs []error errs []error
isFull bool isFull bool
@@ -677,14 +677,14 @@ type Context struct {
genresMissing int genresMissing int
} }
func (c *Context) SeenTracks() int { return len(c.seenTracks) } func (s *State) SeenTracks() int { return len(s.seenTracks) }
func (c *Context) SeenAlbums() int { return len(c.seenAlbums) } func (s *State) SeenAlbums() int { return len(s.seenAlbums) }
func (c *Context) SeenTracksNew() int { return c.seenTracksNew } func (s *State) SeenTracksNew() int { return s.seenTracksNew }
func (c *Context) TracksMissing() int { return len(c.tracksMissing) } func (s *State) TracksMissing() int { return len(s.tracksMissing) }
func (c *Context) AlbumsMissing() int { return len(c.albumsMissing) } func (s *State) AlbumsMissing() int { return len(s.albumsMissing) }
func (c *Context) ArtistsMissing() int { return c.artistsMissing } func (s *State) ArtistsMissing() int { return s.artistsMissing }
func (c *Context) GenresMissing() int { return c.genresMissing } func (s *State) GenresMissing() int { return s.genresMissing }
type MultiValueMode uint8 type MultiValueMode uint8

14
scanner/scanner_fuzz_test.go
View File

@@ -16,13 +16,13 @@ import (
func FuzzScanner(f *testing.F) { func FuzzScanner(f *testing.F) {
checkDelta := func(assert *assert.Assertions, m *mockfs.MockFS, expSeen, expNew int) { checkDelta := func(assert *assert.Assertions, m *mockfs.MockFS, expSeen, expNew int) {
ctx := m.ScanAndClean() st := m.ScanAndClean()
assert.Equal(ctx.SeenTracks(), expSeen) assert.Equal(st.SeenTracks(), expSeen)
assert.Equal(ctx.SeenTracksNew(), expNew) assert.Equal(st.SeenTracksNew(), expNew)
assert.Equal(ctx.TracksMissing(), 0) assert.Equal(st.TracksMissing(), 0)
assert.Equal(ctx.AlbumsMissing(), 0) assert.Equal(st.AlbumsMissing(), 0)
assert.Equal(ctx.ArtistsMissing(), 0) assert.Equal(st.ArtistsMissing(), 0)
assert.Equal(ctx.GenresMissing(), 0) assert.Equal(st.GenresMissing(), 0)
} }
f.Fuzz(func(t *testing.T, data []byte, seed int64) { f.Fuzz(func(t *testing.T, data []byte, seed int64) {

16
scanner/scanner_test.go
View File

@@ -524,22 +524,22 @@ func TestTagErrors(t *testing.T) {
tags.Error = scanner.ErrReadingTags tags.Error = scanner.ErrReadingTags
}) })
ctx, err := m.ScanAndCleanErr() st, err := m.ScanAndCleanErr()
errs, ok := err.(interface{ Unwrap() []error }) errs, ok := err.(interface{ Unwrap() []error })
assert.True(t, ok) assert.True(t, ok)
assert.ErrorAs(t, err, &errs) assert.ErrorAs(t, err, &errs)
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracksNew()) // we have all tracks bar 2 album contents assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracksNew()) // we have all tracks bar 2 album contents
ctx, err = m.ScanAndCleanErr() st, err = m.ScanAndCleanErr()
errs, ok = err.(interface{ Unwrap() []error }) errs, ok = err.(interface{ Unwrap() []error })
assert.True(t, ok) assert.True(t, ok)
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, 0, ctx.SeenTracksNew()) // we have no new tracks assert.Equal(t, 0, st.SeenTracksNew()) // we have no new tracks
} }
// https://github.com/sentriz/gonic/issues/185#issuecomment-1050092128 // https://github.com/sentriz/gonic/issues/185#issuecomment-1050092128