heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(scanner): add option to use fsnotify based scan watcher (#232)

Browse Source 
* First cut at fsnotify support. Tested on Linux.

* Fixed bug in logging.

* Fixed lint issues.

* Added new scan watcher option to README.md

* Inverted conditional and dedented following code as per PR discussion.

* Changed command line switch and error return on ExecuteWatch() as per GH comments.

* Scan watcher: Removed scan at first start. Modified watcher to set a 10 second timer and then process in bulk.

Co-authored-by: Brian Doherty <brian@hplaptop.dohertyfamily.me>
This commit is contained in:
brian-doherty
2022-10-08 08:53:52 -05:00
committed by sentriz
parent bb83426816
commit ea28ff1df3
6 changed files with 143 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -161,6 +161,7 @@ view the admin UI at http://localhost:4747
| `GONIC_TLS_KEY` | `-tls-key` | **optional** path to a TLS key (enables HTTPS listening) |
| `GONIC_PROXY_PREFIX` | `-proxy-prefix` | **optional** url path prefix to use if behind reverse proxy. eg `/gonic` (see example configs below) |
| `GONIC_SCAN_INTERVAL` | `-scan-interval` | **optional** interval (in minutes) to check for new music (automatic scanning disabled if omitted) |
| `GONIC_SCAN_WATCHER ` | `-scan-watcher-enabled` | **optional** whether to watch file system for new music and rescan |
| `GONIC_JUKEBOX_ENABLED` | `-jukebox-enabled` | **optional** whether the subsonic [jukebox api](https://airsonic.github.io/docs/jukebox/) should be enabled |
| `GONIC_GENRE_SPLIT` | `-genre-split` | **optional** a string or character to split genre tags on for multi-genre support (eg. `;`) |

4
cmd/gonic/gonic.go
View File

@@ -36,6 +36,7 @@ func main() {
confCachePath := set.String("cache-path", "", "path to cache")
confDBPath := set.String("db-path", "gonic.db", "path to database (optional)")
confScanInterval := set.Int("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)")
confScanWatcher := set.Bool("scan-watcher-enabled", false, "whether to watch file system for new music and rescan (optional)")
confJukeboxEnabled := set.Bool("jukebox-enabled", false, "whether the subsonic jukebox api should be enabled (optional)")
confProxyPrefix := set.String("proxy-prefix", "", "url path prefix to use if behind proxy. eg '/gonic' (optional)")
confGenreSplit := set.String("genre-split", "\n", "character or string to split genre tag data on (optional)")
@@ -134,6 +135,9 @@ func main() {
tickerDur := time.Duration(*confScanInterval) * time.Minute
g.Add(server.StartScanTicker(tickerDur))
}
if *confScanWatcher {
g.Add(server.StartScanWatcher())
}
if *confJukeboxEnabled {
g.Add(server.StartJukebox())
}

1
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.0
github.com/faiface/beep v1.1.0
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

3
go.sum
View File

@@ -35,6 +35,8 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DP
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/faiface/beep v1.1.0 h1:A2gWP6xf5Rh7RG/p9/VAW2jRSDEGQm5sbOb38sf5d4c=
github.com/faiface/beep v1.1.0/go.mod h1:6I8p6kK2q4opL/eWb+kAkk38ehnTunWeToJB+s51sT4=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
@@ -258,6 +260,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s=
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

126
scanner/scanner.go
View File

@@ -14,6 +14,7 @@ import (
"syscall"
"time"
"github.com/fsnotify/fsnotify"
"github.com/jinzhu/gorm"
"github.com/rainycape/unidecode"
@@ -34,6 +35,9 @@ type Scanner struct {
genreSplit string
tagger tags.Reader
scanning *int32
watcher *fsnotify.Watcher
watchMap map[string]string // maps watched dirs back to root music dir
watchDone chan bool
}
func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
@@ -43,23 +47,30 @@ func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *
genreSplit: genreSplit,
tagger: tagger,
scanning: new(int32),
watchMap: make(map[string]string),
watchDone: make(chan bool),
}
}
func (s *Scanner) IsScanning() bool {
return atomic.LoadInt32(s.scanning) == 1
}
func (s *Scanner) StartScanning() bool {
return atomic.CompareAndSwapInt32(s.scanning, 0, 1)
}
func (s *Scanner) StopScanning() {
defer atomic.StoreInt32(s.scanning, 0)
}
type ScanOptions struct {
IsFull bool
}
func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
if s.IsScanning() {
if !s.StartScanning() {
return nil, ErrAlreadyScanning
}
atomic.StoreInt32(s.scanning, 1)
defer atomic.StoreInt32(s.scanning, 0)
defer s.StopScanning()
start := time.Now()
c := &Context{
@@ -108,6 +119,115 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
return c, nil
}
func (s *Scanner) ExecuteWatch() error {
var err error
s.watcher, err = fsnotify.NewWatcher()
if err != nil {
log.Printf("error creating watcher: %v\n", err)
return err
}
defer s.watcher.Close()
t := time.NewTimer(10 * time.Second)
if !t.Stop() {
<-t.C
}
for _, dir := range s.musicDirs {
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
return s.watchCallback(dir, absPath, d, err)
})
if err != nil {
log.Printf("error watching directory tree: %v\n", err)
}
}
scanList := map[string]struct{}{}
for {
select {
case <-t.C:
if !s.StartScanning() {
scanList = map[string]struct{}{}
break
}
for dirName := range scanList {
c := &Context{
errs: &multierr.Err{},
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
isFull: false,
}
musicDirName := s.watchMap[dirName]
if musicDirName == "" {
musicDirName = s.watchMap[filepath.Dir(dirName)]
}
err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error {
return s.watchCallback(musicDirName, absPath, d, err)
})
if err != nil {
log.Printf("error watching directory tree: %v\n", err)
}
err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, musicDirName, absPath, d, err)
})
if err != nil {
log.Printf("error walking: %v", err)
}
}
scanList = nil
s.StopScanning()
case event := <-s.watcher.Events:
var dirName string
if event.Op&(fsnotify.Create|fsnotify.Write) == 0 {
break
}
if len(scanList) == 0 {
t.Reset(10 * time.Second)
}
fileInfo, err := os.Stat(event.Name)
if err != nil && fileInfo.IsDir() {
dirName = event.Name
} else {
dirName = filepath.Dir(event.Name)
}
scanList[dirName] = struct{}{}
case err = <-s.watcher.Errors:
log.Printf("error from watcher: %v\n", err)
case <-s.watchDone:
return nil
}
}
}
func (s *Scanner) CancelWatch() {
s.watchDone <- true
}
func (s *Scanner) watchCallback(dir string, absPath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
switch d.Type() {
case os.ModeDir:
case os.ModeSymlink:
eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.watchCallback(dir, subAbs, d, err)
})
default:
return nil
}
if s.watchMap[absPath] == "" {
s.watchMap[absPath] = dir
err = s.watcher.Add(absPath)
}
return err
}
func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirEntry, err error) error {
if err != nil {
c.errs.Add(err)

11
server/server.go
View File

@@ -296,6 +296,7 @@ func (s *Server) StartHTTP(listenAddr string, tlsCert string, tlsKey string) (Fu
Addr: listenAddr,
Handler: s.router,
ReadTimeout: 5 * time.Second,
ReadHeaderTimeout: 5 * time.Second,
WriteTimeout: 80 * time.Second,
IdleTimeout: 60 * time.Second,
}
@@ -338,6 +339,16 @@ func (s *Server) StartScanTicker(dur time.Duration) (FuncExecute, FuncInterrupt)
}
}
func (s *Server) StartScanWatcher() (FuncExecute, FuncInterrupt) {
return func() error {
log.Printf("starting job 'scan watcher'\n")
return s.scanner.ExecuteWatch()
}, func(_ error) {
// stop job
s.scanner.CancelWatch()
}
}
func (s *Server) StartJukebox() (FuncExecute, FuncInterrupt) {
return func() error {
log.Printf("starting job 'jukebox'\n")