add "-full-scan" scan option to gonic scan

closes #20
This commit is contained in:
sentriz
2020-04-24 01:42:26 +01:00
parent 5bd57c42d5
commit b95f98ffcf
8 changed files with 695 additions and 684 deletions

View File

@@ -15,12 +15,9 @@ import (
func main() { func main() {
set := flag.NewFlagSet(version.NAME_SCAN, flag.ExitOnError) set := flag.NewFlagSet(version.NAME_SCAN, flag.ExitOnError)
musicPath := set.String( musicPath := set.String("music-path", "", "path to music")
"music-path", "", dbPath := set.String("db-path", "gonic.db", "path to database (optional)")
"path to music") fullScan := set.Bool("full-scan", false, "ignore file modtimes while scanning (optional)")
dbPath := set.String(
"db-path", "gonic.db",
"path to database (optional)")
if err := ff.Parse(set, os.Args[1:], if err := ff.Parse(set, os.Args[1:],
ff.WithEnvVarPrefix(version.NAME_UPPER), ff.WithEnvVarPrefix(version.NAME_UPPER),
); err != nil { ); err != nil {
@@ -34,11 +31,15 @@ func main() {
log.Fatalf("error opening database: %v\n", err) log.Fatalf("error opening database: %v\n", err)
} }
defer db.Close() defer db.Close()
s := scanner.New( s := scanner.New(*musicPath, db)
*musicPath, var scan func() error
db, switch {
) case *fullScan:
if err := s.Start(); err != nil { scan = s.StartFull
log.Fatalf("error starting scanner: %v\n", err) default:
scan = s.StartInc
}
if err := scan(); err != nil {
log.Fatalf("error during scan: %v\n", err)
} }
} }

View File

@@ -39,6 +39,7 @@ func SetScanning() func() {
type Scanner struct { type Scanner struct {
db *db.DB db *db.DB
musicPath string musicPath string
isFull bool
// these two are for the transaction we do for every folder. // these two are for the transaction we do for every folder.
// the boolean is there so we dont begin or commit multiple // the boolean is there so we dont begin or commit multiple
// times in the handle folder or post children callback // times in the handle folder or post children callback
@@ -67,7 +68,7 @@ func New(musicPath string, db *db.DB) *Scanner {
} }
} }
func (s *Scanner) Start() error { func (s *Scanner) Start(isFull bool) error {
if IsScanning() { if IsScanning() {
return errors.New("already scanning") return errors.New("already scanning")
} }
@@ -82,6 +83,7 @@ func (s *Scanner) Start() error {
s.seenTracksErr = 0 s.seenTracksErr = 0
}() }()
// ** begin being walking // ** begin being walking
s.isFull = isFull
start := time.Now() start := time.Now()
err := godirwalk.Walk(s.musicPath, &godirwalk.Options{ err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
Callback: s.callbackItem, Callback: s.callbackItem,
@@ -146,6 +148,14 @@ func (s *Scanner) Start() error {
return nil return nil
} }
func (s *Scanner) StartInc() error {
return s.Start(false)
}
func (s *Scanner) StartFull() error {
return s.Start(true)
}
// items are passed to the handle*() functions // items are passed to the handle*() functions
type item struct { type item struct {
fullPath string fullPath string
@@ -274,8 +284,8 @@ func (s *Scanner) handleFolder(it *item) error {
}). }).
First(folder). First(folder).
Error Error
if !gorm.IsRecordNotFoundError(err) && if !s.isFull && (!gorm.IsRecordNotFoundError(err) &&
it.stat.ModTime().Before(folder.UpdatedAt) { it.stat.ModTime().Before(folder.UpdatedAt)) {
// we found the record but it hasn't changed // we found the record but it hasn't changed
return nil return nil
} }
@@ -303,7 +313,7 @@ func (s *Scanner) handleTrack(it *item) error {
}). }).
First(track). First(track).
Error Error
if !gorm.IsRecordNotFoundError(err) && if !s.isFull && !gorm.IsRecordNotFoundError(err) &&
it.stat.ModTime().Before(track.UpdatedAt) { it.stat.ModTime().Before(track.UpdatedAt) {
// we found the record but it hasn't changed // we found the record but it hasn't changed
s.seenTracks[track.ID] = struct{}{} s.seenTracks[track.ID] = struct{}{}

View File

@@ -40,17 +40,17 @@ func resetTablesPause(db *db.DB, b *testing.B) {
func BenchmarkScanFresh(b *testing.B) { func BenchmarkScanFresh(b *testing.B) {
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
resetTablesPause(testScanner.db, b) resetTablesPause(testScanner.db, b)
testScanner.Start() testScanner.StartInc()
} }
} }
func BenchmarkScanIncremental(b *testing.B) { func BenchmarkScanIncremental(b *testing.B) {
// do a full scan and reset // do a full scan and reset
testScanner.Start() testScanner.StartInc()
b.ResetTimer() b.ResetTimer()
// do the inc scans // do the inc scans
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
testScanner.Start() testScanner.StartInc()
} }
} }

File diff suppressed because it is too large Load Diff

View File

@@ -272,7 +272,7 @@ func (c *Controller) ServeUpdateLastFMAPIKeyDo(r *http.Request) *Response {
func (c *Controller) ServeStartScanDo(r *http.Request) *Response { func (c *Controller) ServeStartScanDo(r *http.Request) *Response {
defer func() { defer func() {
go func() { go func() {
if err := c.Scanner.Start(); err != nil { if err := c.Scanner.StartInc(); err != nil {
log.Printf("error while scanning: %v\n", err) log.Printf("error while scanning: %v\n", err)
} }
}() }()

View File

@@ -86,7 +86,7 @@ func (c *Controller) ServeGetMusicFolders(r *http.Request) *spec.Response {
func (c *Controller) ServeStartScan(r *http.Request) *spec.Response { func (c *Controller) ServeStartScan(r *http.Request) *spec.Response {
go func() { go func() {
if err := c.Scanner.Start(); err != nil { if err := c.Scanner.StartInc(); err != nil {
log.Printf("error while scanning: %v\n", err) log.Printf("error while scanning: %v\n", err)
} }
}() }()

Binary file not shown.

View File

@@ -208,7 +208,7 @@ func (s *Server) StartScanTicker(dur time.Duration) (funcExecute, funcInterrupt)
case <-done: case <-done:
return nil return nil
case <-ticker.C: case <-ticker.C:
if err := s.scanner.Start(); err != nil { if err := s.scanner.StartInc(); err != nil {
log.Printf("error scanning: %v", err) log.Printf("error scanning: %v", err)
} }
} }