feat(scanner): add a new option for excluding paths based on a regexp

* Exclude paths based on new exclude pattern option

* Add test for excluded paths

* Add exclude pattern option to docs

* Set exclude regexp only if given argument is set

* Update scanner/scanner.go

---------

Co-authored-by: Senan Kelly <senan@senan.xyz>

README.md

@@ -73,6 +73,7 @@ password can then be changed from the web interface
 | `GONIC_JUKEBOX_MPV_EXTRA_ARGS` | `-jukebox-mpv-extra-args` | **optional** extra command line arguments to pass to the jukebox mpv daemon                                 |
 | `GONIC_PODCAST_PURGE_AGE`      | `-podcast-purge-age`      | **optional** age (in days) to purge podcast episodes if not accessed                                        |
 | `GONIC_GENRE_SPLIT`            | `-genre-split`            | **optional** a string or character to split genre tags on for multi-genre support (eg. `;`)                 |
+| `GONIC_EXCLUDE_PATTERN`        | `-exclude-pattern`        | **optional** files matching this regex pattern will not be imported                                         |
 
 ## screenshots
 

cmd/gonic/gonic.go

@@ -62,6 +62,12 @@ func main() {
 	confShowVersion := set.Bool("version", false, "show gonic version")
 	_ = set.String("config-path", "", "path to config (optional)")
 
+	confExcludePatterns := set.String("exclude-pattern", "", "regex pattern to exclude files from scan (optional)")
+
+	if _, err := regexp.Compile(*confExcludePatterns); err != nil {
+		log.Fatalf("invalid exclude pattern: %v\n", err)
+	}
+
 	if err := ff.Parse(set, os.Args[1:],
 		ff.WithConfigFileFlag("config-path"),
 		ff.WithConfigFileParser(ff.PlainParser),
@@ -125,6 +131,7 @@ func main() {
 	server, err := server.New(server.Options{
 		DB:             dbc,
 		MusicPaths:     musicPaths,
+		ExcludePattern: *confExcludePatterns,
 		CacheAudioPath: cacheDirAudio,
 		CoverCachePath: cacheDirCovers,
 		PodcastPath:    *confPodcastPath,

mockfs/mockfs.go

@@ -27,10 +27,13 @@ type MockFS struct {
 	db        *db.DB
 }
 
-func New(t testing.TB) *MockFS                        { return new(t, []string{""}) }
-func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs) }
+func New(t testing.TB) *MockFS                        { return new(t, []string{""}, "") }
+func NewWithDirs(t testing.TB, dirs []string) *MockFS { return new(t, dirs, "") }
+func NewWithExcludePattern(t testing.TB, excludePattern string) *MockFS {
+	return new(t, []string{""}, excludePattern)
+}
 
-func new(t testing.TB, dirs []string) *MockFS {
+func new(t testing.TB, dirs []string, excludePattern string) *MockFS {
 	dbc, err := db.NewMock()
 	if err != nil {
 		t.Fatalf("create db: %v", err)
@@ -59,7 +62,7 @@ func new(t testing.TB, dirs []string) *MockFS {
 	}
 
 	tagReader := &tagReader{paths: map[string]*tagReaderResult{}}
-	scanner := scanner.New(absDirs, dbc, ";", tagReader)
+	scanner := scanner.New(absDirs, dbc, ";", tagReader, excludePattern)
 
 	return &MockFS{
 		t:         t,

scanner/scanner.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -29,25 +30,32 @@ var (
 )
 
 type Scanner struct {
-	db         *db.DB
-	musicDirs  []string
-	genreSplit string
-	tagger     tags.Reader
-	scanning   *int32
-	watcher    *fsnotify.Watcher
-	watchMap   map[string]string // maps watched dirs back to root music dir
-	watchDone  chan bool
+	db             *db.DB
+	musicDirs      []string
+	genreSplit     string
+	tagger         tags.Reader
+	excludePattern *regexp.Regexp
+	scanning       *int32
+	watcher        *fsnotify.Watcher
+	watchMap       map[string]string // maps watched dirs back to root music dir
+	watchDone      chan bool
 }
 
-func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
+func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader, excludePattern string) *Scanner {
+	var excludePatternRegExp *regexp.Regexp
+	if excludePattern != "" {
+		excludePatternRegExp = regexp.MustCompile(excludePattern)
+	}
+
 	return &Scanner{
-		db:         db,
-		musicDirs:  musicDirs,
-		genreSplit: genreSplit,
-		tagger:     tagger,
-		scanning:   new(int32),
-		watchMap:   make(map[string]string),
-		watchDone:  make(chan bool),
+		db:             db,
+		musicDirs:      musicDirs,
+		genreSplit:     genreSplit,
+		tagger:         tagger,
+		excludePattern: excludePatternRegExp,
+		scanning:       new(int32),
+		watchMap:       make(map[string]string),
+		watchDone:      make(chan bool),
 	}
 }
 
@@ -251,6 +259,11 @@ func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirE
 		return nil
 	}
 
+	if s.excludePattern != nil && s.excludePattern.MatchString(absPath) {
+		log.Printf("excluding folder `%s`", absPath)
+		return nil
+	}
+
 	log.Printf("processing folder `%s`", absPath)
 
 	tx := s.db.Begin()
@@ -275,6 +288,12 @@ func (s *Scanner) scanDir(tx *db.DB, c *Context, musicDir string, absPath string
 	var tracks []string
 	var cover string
 	for _, item := range items {
+		fullpath := filepath.Join(absPath, item.Name())
+		if s.excludePattern != nil && s.excludePattern.MatchString(fullpath) {
+			log.Printf("excluding path `%s`", fullpath)
+			continue
+		}
+
 		if isCover(item.Name()) {
 			cover = item.Name()
 			continue

scanner/scanner_test.go

@@ -45,6 +45,27 @@ func TestTableCounts(t *testing.T) {
 	is.Equal(artists, 3)                                       // not all artists
 }
 
+func TestWithExcludePattern(t *testing.T) {
+	t.Parallel()
+	is := is.NewRelaxed(t)
+	m := mockfs.NewWithExcludePattern(t, "\\/artist-1\\/|track-0.flac$")
+
+	m.AddItems()
+	m.ScanAndClean()
+
+	var tracks int
+	is.NoErr(m.DB().Model(&db.Track{}).Count(&tracks).Error) // not all tracks
+	is.Equal(tracks, 12)
+
+	var albums int
+	is.NoErr(m.DB().Model(&db.Album{}).Count(&albums).Error) // not all albums
+	is.Equal(albums, 10)                                     // not all albums
+
+	var artists int
+	is.NoErr(m.DB().Model(&db.Artist{}).Count(&artists).Error) // not all artists
+	is.Equal(artists, 2)                                       // not all artists
+}
+
 func TestParentID(t *testing.T) {
 	t.Parallel()
 	is := is.New(t)

server/server.go

@@ -31,6 +31,7 @@ import (
 type Options struct {
 	DB             *db.DB
 	MusicPaths     []ctrlsubsonic.MusicPath
+	ExcludePattern string
 	PodcastPath    string
 	CacheAudioPath string
 	CoverCachePath string
@@ -51,7 +52,7 @@ type Server struct {
 func New(opts Options) (*Server, error) {
 	tagger := &tags.TagReader{}
 
-	scanner := scanner.New(ctrlsubsonic.PathsOf(opts.MusicPaths), opts.DB, opts.GenreSplit, tagger)
+	scanner := scanner.New(ctrlsubsonic.PathsOf(opts.MusicPaths), opts.DB, opts.GenreSplit, tagger, opts.ExcludePattern)
 	base := &ctrlbase.Controller{
 		DB:          opts.DB,
 		ProxyPrefix: opts.ProxyPrefix,
@@ -94,7 +95,7 @@ func New(opts Options) (*Server, error) {
 	ctrlSubsonic := &ctrlsubsonic.Controller{
 		Controller:     base,
 		MusicPaths:     opts.MusicPaths,
-		PodcastsPath:    opts.PodcastPath,
+		PodcastsPath:   opts.PodcastPath,
 		CacheAudioPath: opts.CacheAudioPath,
 		CoverCachePath: opts.CoverCachePath,
 		Scrobblers:     []scrobble.Scrobbler{&lastfm.Scrobbler{DB: opts.DB}, &listenbrainz.Scrobbler{}},