From 1d3877668f9bf925cae7c548b5d8e683d9676af2 Mon Sep 17 00:00:00 2001 From: Gregor Zurowski Date: Sat, 6 May 2023 19:03:11 +0200 Subject: [PATCH] feat(scanner): add a new option for excluding paths based on a regexp * Exclude paths based on new exclude pattern option * Add test for excluded paths * Add exclude pattern option to docs * Set exclude regexp only if given argument is set * Update scanner/scanner.go --------- Co-authored-by: Senan Kelly --- README.md | 1 + cmd/gonic/gonic.go | 7 ++++++ mockfs/mockfs.go | 11 +++++---- scanner/scanner.go | 51 ++++++++++++++++++++++++++++------------- scanner/scanner_test.go | 21 +++++++++++++++++ server/server.go | 5 ++-- 6 files changed, 74 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 499fb1b..228ad05 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ password can then be changed from the web interface | `GONIC_JUKEBOX_MPV_EXTRA_ARGS` | `-jukebox-mpv-extra-args` | **optional** extra command line arguments to pass to the jukebox mpv daemon | | `GONIC_PODCAST_PURGE_AGE` | `-podcast-purge-age` | **optional** age (in days) to purge podcast episodes if not accessed | | `GONIC_GENRE_SPLIT` | `-genre-split` | **optional** a string or character to split genre tags on for multi-genre support (eg. `;`) | +| `GONIC_EXCLUDE_PATTERN` | `-exclude-pattern` | **optional** files matching this regex pattern will not be imported | ## screenshots diff --git a/cmd/gonic/gonic.go b/cmd/gonic/gonic.go index b5b97d3..db302ae 100644 --- a/cmd/gonic/gonic.go +++ b/cmd/gonic/gonic.go @@ -62,6 +62,12 @@ func main() { confShowVersion := set.Bool("version", false, "show gonic version") _ = set.String("config-path", "", "path to config (optional)") + confExcludePatterns := set.String("exclude-pattern", "", "regex pattern to exclude files from scan (optional)") + + if _, err := regexp.Compile(*confExcludePatterns); err != nil { + log.Fatalf("invalid exclude pattern: %v\n", err) + } + if err := ff.Parse(set, os.Args[1:], ff.WithConfigFileFlag("config-path"), ff.WithConfigFileParser(ff.PlainParser), @@ -125,6 +131,7 @@ func main() { server, err := server.New(server.Options{ DB: dbc, MusicPaths: musicPaths, + ExcludePattern: *confExcludePatterns, CacheAudioPath: cacheDirAudio, CoverCachePath: cacheDirCovers, PodcastPath: *confPodcastPath, diff --git a/mockfs/mockfs.go b/mockfs/mockfs.go index a3fdc3e..cd16649 100644 --- a/mockfs/mockfs.go +++ b/mockfs/mockfs.go @@ -27,10 +27,13 @@ type MockFS struct { db *db.DB } -func New(t testing.TB) *MockFS { return new(t, []string{""}) } -func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs) } +func New(t testing.TB) *MockFS { return new(t, []string{""}, "") } +func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs, "") } +func NewWithExcludePattern(t testing.TB, excludePattern string) *MockFS { + return new(t, []string{""}, excludePattern) +} -func new(t testing.TB, dirs []string) *MockFS { +func new(t testing.TB, dirs []string, excludePattern string) *MockFS { dbc, err := db.NewMock() if err != nil { t.Fatalf("create db: %v", err) @@ -59,7 +62,7 @@ func new(t testing.TB, dirs []string) *MockFS { } tagReader := &tagReader{paths: map[string]*tagReaderResult{}} - scanner := scanner.New(absDirs, dbc, ";", tagReader) + scanner := scanner.New(absDirs, dbc, ";", tagReader, excludePattern) return &MockFS{ t: t, diff --git a/scanner/scanner.go b/scanner/scanner.go index 10d52fd..5eabefa 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -7,6 +7,7 @@ import ( "log" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -29,25 +30,32 @@ var ( ) type Scanner struct { - db *db.DB - musicDirs []string - genreSplit string - tagger tags.Reader - scanning *int32 - watcher *fsnotify.Watcher - watchMap map[string]string // maps watched dirs back to root music dir - watchDone chan bool + db *db.DB + musicDirs []string + genreSplit string + tagger tags.Reader + excludePattern *regexp.Regexp + scanning *int32 + watcher *fsnotify.Watcher + watchMap map[string]string // maps watched dirs back to root music dir + watchDone chan bool } -func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner { +func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader, excludePattern string) *Scanner { + var excludePatternRegExp *regexp.Regexp + if excludePattern != "" { + excludePatternRegExp = regexp.MustCompile(excludePattern) + } + return &Scanner{ - db: db, - musicDirs: musicDirs, - genreSplit: genreSplit, - tagger: tagger, - scanning: new(int32), - watchMap: make(map[string]string), - watchDone: make(chan bool), + db: db, + musicDirs: musicDirs, + genreSplit: genreSplit, + tagger: tagger, + excludePattern: excludePatternRegExp, + scanning: new(int32), + watchMap: make(map[string]string), + watchDone: make(chan bool), } } @@ -251,6 +259,11 @@ func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirE return nil } + if s.excludePattern != nil && s.excludePattern.MatchString(absPath) { + log.Printf("excluding folder `%s`", absPath) + return nil + } + log.Printf("processing folder `%s`", absPath) tx := s.db.Begin() @@ -275,6 +288,12 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, musicDir string, absPath string var tracks []string var cover string for _, item := range items { + fullpath := filepath.Join(absPath, item.Name()) + if s.excludePattern != nil && s.excludePattern.MatchString(fullpath) { + log.Printf("excluding path `%s`", fullpath) + continue + } + if isCover(item.Name()) { cover = item.Name() continue diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 39dc135..214377f 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -45,6 +45,27 @@ func TestTableCounts(t *testing.T) { is.Equal(artists, 3) // not all artists } +func TestWithExcludePattern(t *testing.T) { + t.Parallel() + is := is.NewRelaxed(t) + m := mockfs.NewWithExcludePattern(t, "\\/artist-1\\/|track-0.flac$") + + m.AddItems() + m.ScanAndClean() + + var tracks int + is.NoErr(m.DB().Model(&db.Track{}).Count(&tracks).Error) // not all tracks + is.Equal(tracks, 12) + + var albums int + is.NoErr(m.DB().Model(&db.Album{}).Count(&albums).Error) // not all albums + is.Equal(albums, 10) // not all albums + + var artists int + is.NoErr(m.DB().Model(&db.Artist{}).Count(&artists).Error) // not all artists + is.Equal(artists, 2) // not all artists +} + func TestParentID(t *testing.T) { t.Parallel() is := is.New(t) diff --git a/server/server.go b/server/server.go index a88d020..b688b98 100644 --- a/server/server.go +++ b/server/server.go @@ -31,6 +31,7 @@ import ( type Options struct { DB *db.DB MusicPaths []ctrlsubsonic.MusicPath + ExcludePattern string PodcastPath string CacheAudioPath string CoverCachePath string @@ -51,7 +52,7 @@ type Server struct { func New(opts Options) (*Server, error) { tagger := &tags.TagReader{} - scanner := scanner.New(ctrlsubsonic.PathsOf(opts.MusicPaths), opts.DB, opts.GenreSplit, tagger) + scanner := scanner.New(ctrlsubsonic.PathsOf(opts.MusicPaths), opts.DB, opts.GenreSplit, tagger, opts.ExcludePattern) base := &ctrlbase.Controller{ DB: opts.DB, ProxyPrefix: opts.ProxyPrefix, @@ -94,7 +95,7 @@ func New(opts Options) (*Server, error) { ctrlSubsonic := &ctrlsubsonic.Controller{ Controller: base, MusicPaths: opts.MusicPaths, - PodcastsPath: opts.PodcastPath, + PodcastsPath: opts.PodcastPath, CacheAudioPath: opts.CacheAudioPath, CoverCachePath: opts.CoverCachePath, Scrobblers: []scrobble.Scrobbler{&lastfm.Scrobbler{DB: opts.DB}, &listenbrainz.Scrobbler{}},