heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(scanner): rename scan context to scan state

Browse Source
This commit is contained in:
sentriz
2023-12-13 01:32:01 +00:00
parent a892595641
commit 9e12394acc
4 changed files with 68 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
scanner/scanner.go
View File

@@ -61,14 +61,14 @@ type ScanOptions struct {
IsFull bool
}
func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
func (s *Scanner) ScanAndClean(opts ScanOptions) (*State, error) {
if !s.StartScanning() {
return nil, ErrAlreadyScanning
}
defer s.StopScanning()
start := time.Now()
c := &Context{
st := &State{
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
isFull: opts.IsFull,
@@ -77,28 +77,28 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.SeenTracksNew(), c.SeenTracks(), len(c.errs))
durSince(start), st.SeenTracksNew(), st.SeenTracks(), len(st.errs))
}()
for _, dir := range s.musicDirs {
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, absPath, d, err)
return s.scanCallback(st, absPath, d, err)
})
if err != nil {
return nil, fmt.Errorf("walk: %w", err)
}
}
if err := s.cleanTracks(c); err != nil {
if err := s.cleanTracks(st); err != nil {
return nil, fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(c); err != nil {
if err := s.cleanAlbums(st); err != nil {
return nil, fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(c); err != nil {
if err := s.cleanArtists(st); err != nil {
return nil, fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(c); err != nil {
if err := s.cleanGenres(st); err != nil {
return nil, fmt.Errorf("clean genres: %w", err)
}
@@ -106,7 +106,7 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
return nil, fmt.Errorf("set scan time: %w", err)
}
return c, errors.Join(c.errs...)
return st, errors.Join(st.errs...)
}
func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
@@ -138,7 +138,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
break
}
for absPath := range batchSeen {
c := &Context{
st := &State{
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
}
@@ -150,7 +150,7 @@ func (s *Scanner) ExecuteWatch(done <-chan struct{}) error {
continue
}
err = filepath.WalkDir(absPath, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, absPath, d, err)
return s.scanCallback(st, absPath, d, err)
})
if err != nil {
log.Printf("error walking: %v", err)
@@ -207,9 +207,9 @@ func watchCallback(watcher *fsnotify.Watcher, absPath string, d fs.DirEntry, err
return nil
}
func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err error) error {
func (s *Scanner) scanCallback(st *State, absPath string, d fs.DirEntry, err error) error {
if err != nil {
c.errs = append(c.errs, err)
st.errs = append(st.errs, err)
return nil
}
@@ -219,7 +219,7 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er
eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.scanCallback(c, subAbs, d, err)
return s.scanCallback(st, subAbs, d, err)
})
default:
return nil
@@ -233,15 +233,15 @@ func (s *Scanner) scanCallback(c *Context, absPath string, d fs.DirEntry, err er
log.Printf("processing folder %q", absPath)
return s.db.Transaction(func(tx *db.DB) error {
if err := s.scanDir(tx, c, absPath); err != nil {
c.errs = append(c.errs, fmt.Errorf("%q: %w", absPath, err))
if err := s.scanDir(tx, st, absPath); err != nil {
st.errs = append(st.errs, fmt.Errorf("%q: %w", absPath, err))
return nil
}
return nil
})
}
func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
func (s *Scanner) scanDir(tx *db.DB, st *State, absPath string) error {
musicDir, relPath := musicDirRelative(s.musicDirs, absPath)
if musicDir == absPath {
return nil
@@ -280,7 +280,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return fmt.Errorf("first or create parent: %w", err)
}
c.seenAlbums[parent.ID] = struct{}{}
st.seenAlbums[parent.ID] = struct{}{}
dir, basename := filepath.Split(relPath)
var album db.Album
@@ -288,12 +288,12 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return fmt.Errorf("populate album basics: %w", err)
}
c.seenAlbums[album.ID] = struct{}{}
st.seenAlbums[album.ID] = struct{}{}
sort.Strings(tracks)
for i, basename := range tracks {
absPath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndArtists(tx, c, i, &album, basename, absPath); err != nil {
if err := s.populateTrackAndArtists(tx, st, i, &album, basename, absPath); err != nil {
return fmt.Errorf("populate track %q: %w", basename, err)
}
}
@@ -301,7 +301,7 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, absPath string) error {
return nil
}
func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *db.Album, basename string, absPath string) error {
func (s *Scanner) populateTrackAndArtists(tx *db.DB, st *State, i int, album *db.Album, basename string, absPath string) error {
stat, err := os.Stat(absPath)
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
@@ -312,8 +312,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d
return fmt.Errorf("query track: %w", err)
}
if !c.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) {
c.seenTracks[track.ID] = struct{}{}
if !st.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) {
st.seenTracks[track.ID] = struct{}{}
return nil
}
@@ -384,8 +384,8 @@ func (s *Scanner) populateTrackAndArtists(tx *db.DB, c *Context, i int, album *d
return fmt.Errorf("populate track artists: %w", err)
}
c.seenTracks[track.ID] = struct{}{}
c.seenTracksNew++
st.seenTracks[track.ID] = struct{}{}
st.seenTracksNew++
return nil
}
@@ -540,9 +540,9 @@ func populateArtistAppearances(tx *db.DB, album *db.Album, artistIDs []int) erro
return nil
}
func (s *Scanner) cleanTracks(c *Context) error {
func (s *Scanner) cleanTracks(st *State) error {
start := time.Now()
defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), c.TracksMissing()) }()
defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), st.TracksMissing()) }()
var all []int
err := s.db.
@@ -553,18 +553,18 @@ func (s *Scanner) cleanTracks(c *Context) error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, a := range all {
if _, ok := c.seenTracks[a]; !ok {
c.tracksMissing = append(c.tracksMissing, int64(a))
if _, ok := st.seenTracks[a]; !ok {
st.tracksMissing = append(st.tracksMissing, int64(a))
}
}
return s.db.TransactionChunked(c.tracksMissing, func(tx *db.DB, chunk []int64) error {
return s.db.TransactionChunked(st.tracksMissing, func(tx *db.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Track{}).Error
})
}
func (s *Scanner) cleanAlbums(c *Context) error {
func (s *Scanner) cleanAlbums(st *State) error {
start := time.Now()
defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), c.AlbumsMissing()) }()
defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), st.AlbumsMissing()) }()
var all []int
err := s.db.
@@ -575,18 +575,18 @@ func (s *Scanner) cleanAlbums(c *Context) error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, a := range all {
if _, ok := c.seenAlbums[a]; !ok {
c.albumsMissing = append(c.albumsMissing, int64(a))
if _, ok := st.seenAlbums[a]; !ok {
st.albumsMissing = append(st.albumsMissing, int64(a))
}
}
return s.db.TransactionChunked(c.albumsMissing, func(tx *db.DB, chunk []int64) error {
return s.db.TransactionChunked(st.albumsMissing, func(tx *db.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Album{}).Error
})
}
func (s *Scanner) cleanArtists(c *Context) error {
func (s *Scanner) cleanArtists(st *State) error {
start := time.Now()
defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), c.ArtistsMissing()) }()
defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), st.ArtistsMissing()) }()
// gorm doesn't seem to support subqueries without parens for UNION
q := s.db.Exec(`
@@ -602,13 +602,13 @@ func (s *Scanner) cleanArtists(c *Context) error {
if err := q.Error; err != nil {
return err
}
c.artistsMissing = int(q.RowsAffected)
st.artistsMissing = int(q.RowsAffected)
return nil
}
func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam
func (s *Scanner) cleanGenres(st *State) error { //nolint:unparam
start := time.Now()
defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), c.GenresMissing()) }()
defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), st.GenresMissing()) }()
subTrack := s.db.
Select("genres.id").
@@ -625,7 +625,7 @@ func (s *Scanner) cleanGenres(c *Context) error { //nolint:unparam
q := s.db.
Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum).
Delete(&db.Genre{})
c.genresMissing = int(q.RowsAffected)
st.genresMissing = int(q.RowsAffected)
return nil
}
@@ -663,7 +663,7 @@ func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
type Context struct {
type State struct {
errs []error
isFull bool
@@ -677,14 +677,14 @@ type Context struct {
genresMissing int
}
func (c *Context) SeenTracks() int { return len(c.seenTracks) }
func (c *Context) SeenAlbums() int { return len(c.seenAlbums) }
func (c *Context) SeenTracksNew() int { return c.seenTracksNew }
func (s *State) SeenTracks() int { return len(s.seenTracks) }
func (s *State) SeenAlbums() int { return len(s.seenAlbums) }
func (s *State) SeenTracksNew() int { return s.seenTracksNew }
func (c *Context) TracksMissing() int { return len(c.tracksMissing) }
func (c *Context) AlbumsMissing() int { return len(c.albumsMissing) }
func (c *Context) ArtistsMissing() int { return c.artistsMissing }
func (c *Context) GenresMissing() int { return c.genresMissing }
func (s *State) TracksMissing() int { return len(s.tracksMissing) }
func (s *State) AlbumsMissing() int { return len(s.albumsMissing) }
func (s *State) ArtistsMissing() int { return s.artistsMissing }
func (s *State) GenresMissing() int { return s.genresMissing }
type MultiValueMode uint8

14
scanner/scanner_fuzz_test.go
View File

@@ -16,13 +16,13 @@ import (
func FuzzScanner(f *testing.F) {
checkDelta := func(assert *assert.Assertions, m *mockfs.MockFS, expSeen, expNew int) {
ctx := m.ScanAndClean()
assert.Equal(ctx.SeenTracks(), expSeen)
assert.Equal(ctx.SeenTracksNew(), expNew)
assert.Equal(ctx.TracksMissing(), 0)
assert.Equal(ctx.AlbumsMissing(), 0)
assert.Equal(ctx.ArtistsMissing(), 0)
assert.Equal(ctx.GenresMissing(), 0)
st := m.ScanAndClean()
assert.Equal(st.SeenTracks(), expSeen)
assert.Equal(st.SeenTracksNew(), expNew)
assert.Equal(st.TracksMissing(), 0)
assert.Equal(st.AlbumsMissing(), 0)
assert.Equal(st.ArtistsMissing(), 0)
assert.Equal(st.GenresMissing(), 0)
}
f.Fuzz(func(t *testing.T, data []byte, seed int64) {

16
scanner/scanner_test.go
View File

@@ -524,22 +524,22 @@ func TestTagErrors(t *testing.T) {
tags.Error = scanner.ErrReadingTags
})
ctx, err := m.ScanAndCleanErr()
st, err := m.ScanAndCleanErr()
errs, ok := err.(interface{ Unwrap() []error })
assert.True(t, ok)
assert.ErrorAs(t, err, &errs)
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracksNew()) // we have all tracks bar 2 album contents
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracksNew()) // we have all tracks bar 2 album contents
ctx, err = m.ScanAndCleanErr()
st, err = m.ScanAndCleanErr()
errs, ok = err.(interface{ Unwrap() []error })
assert.True(t, ok)
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), ctx.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, 0, ctx.SeenTracksNew()) // we have no new tracks
assert.Equal(t, 2, len(errs.Unwrap())) // we have 2 dir errors
assert.Equal(t, m.NumTracks()-(3*2), st.SeenTracks()) // we saw all tracks bar 2 album contents
assert.Equal(t, 0, st.SeenTracksNew()) // we have no new tracks
}
// https://github.com/sentriz/gonic/issues/185#issuecomment-1050092128