From ea28ff1df3f0ea30c53bc79c2e9980ea7ad7206b Mon Sep 17 00:00:00 2001 From: brian-doherty <76168809+brian-doherty@users.noreply.github.com> Date: Sat, 8 Oct 2022 08:53:52 -0500 Subject: [PATCH] feat(scanner): add option to use fsnotify based scan watcher (#232) * First cut at fsnotify support. Tested on Linux. * Fixed bug in logging. * Fixed lint issues. * Added new scan watcher option to README.md * Inverted conditional and dedented following code as per PR discussion. * Changed command line switch and error return on ExecuteWatch() as per GH comments. * Scan watcher: Removed scan at first start. Modified watcher to set a 10 second timer and then process in bulk. Co-authored-by: Brian Doherty --- README.md | 1 + cmd/gonic/gonic.go | 4 ++ go.mod | 1 + go.sum | 3 ++ scanner/scanner.go | 126 +++++++++++++++++++++++++++++++++++++++++++-- server/server.go | 11 ++++ 6 files changed, 143 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0cc03120..1f3c464d 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ view the admin UI at http://localhost:4747 | `GONIC_TLS_KEY` | `-tls-key` | **optional** path to a TLS key (enables HTTPS listening) | | `GONIC_PROXY_PREFIX` | `-proxy-prefix` | **optional** url path prefix to use if behind reverse proxy. eg `/gonic` (see example configs below) | | `GONIC_SCAN_INTERVAL` | `-scan-interval` | **optional** interval (in minutes) to check for new music (automatic scanning disabled if omitted) | +| `GONIC_SCAN_WATCHER ` | `-scan-watcher-enabled` | **optional** whether to watch file system for new music and rescan | | `GONIC_JUKEBOX_ENABLED` | `-jukebox-enabled` | **optional** whether the subsonic [jukebox api](https://airsonic.github.io/docs/jukebox/) should be enabled | | `GONIC_GENRE_SPLIT` | `-genre-split` | **optional** a string or character to split genre tags on for multi-genre support (eg. `;`) | diff --git a/cmd/gonic/gonic.go b/cmd/gonic/gonic.go index 0d030c64..e9667f19 100644 --- a/cmd/gonic/gonic.go +++ b/cmd/gonic/gonic.go @@ -36,6 +36,7 @@ func main() { confCachePath := set.String("cache-path", "", "path to cache") confDBPath := set.String("db-path", "gonic.db", "path to database (optional)") confScanInterval := set.Int("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)") + confScanWatcher := set.Bool("scan-watcher-enabled", false, "whether to watch file system for new music and rescan (optional)") confJukeboxEnabled := set.Bool("jukebox-enabled", false, "whether the subsonic jukebox api should be enabled (optional)") confProxyPrefix := set.String("proxy-prefix", "", "url path prefix to use if behind proxy. eg '/gonic' (optional)") confGenreSplit := set.String("genre-split", "\n", "character or string to split genre tag data on (optional)") @@ -134,6 +135,9 @@ func main() { tickerDur := time.Duration(*confScanInterval) * time.Minute g.Add(server.StartScanTicker(tickerDur)) } + if *confScanWatcher { + g.Add(server.StartScanWatcher()) + } if *confJukeboxEnabled { g.Add(server.StartJukebox()) } diff --git a/go.mod b/go.mod index 70c7de3b..eb8d0367 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dustin/go-humanize v1.0.0 github.com/faiface/beep v1.1.0 + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.21.1 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go.sum b/go.sum index 4e563f4b..47370494 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DP github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/faiface/beep v1.1.0 h1:A2gWP6xf5Rh7RG/p9/VAW2jRSDEGQm5sbOb38sf5d4c= github.com/faiface/beep v1.1.0/go.mod h1:6I8p6kK2q4opL/eWb+kAkk38ehnTunWeToJB+s51sT4= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs= @@ -258,6 +260,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc= golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/scanner/scanner.go b/scanner/scanner.go index e53e7c9f..4db3a17f 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/fsnotify/fsnotify" "github.com/jinzhu/gorm" "github.com/rainycape/unidecode" @@ -34,6 +35,9 @@ type Scanner struct { genreSplit string tagger tags.Reader scanning *int32 + watcher *fsnotify.Watcher + watchMap map[string]string // maps watched dirs back to root music dir + watchDone chan bool } func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner { @@ -43,23 +47,30 @@ func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) * genreSplit: genreSplit, tagger: tagger, scanning: new(int32), + watchMap: make(map[string]string), + watchDone: make(chan bool), } } func (s *Scanner) IsScanning() bool { return atomic.LoadInt32(s.scanning) == 1 } +func (s *Scanner) StartScanning() bool { + return atomic.CompareAndSwapInt32(s.scanning, 0, 1) +} +func (s *Scanner) StopScanning() { + defer atomic.StoreInt32(s.scanning, 0) +} type ScanOptions struct { IsFull bool } func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { - if s.IsScanning() { + if !s.StartScanning() { return nil, ErrAlreadyScanning } - atomic.StoreInt32(s.scanning, 1) - defer atomic.StoreInt32(s.scanning, 0) + defer s.StopScanning() start := time.Now() c := &Context{ @@ -108,6 +119,115 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) { return c, nil } +func (s *Scanner) ExecuteWatch() error { + var err error + s.watcher, err = fsnotify.NewWatcher() + if err != nil { + log.Printf("error creating watcher: %v\n", err) + return err + } + defer s.watcher.Close() + + t := time.NewTimer(10 * time.Second) + if !t.Stop() { + <-t.C + } + + for _, dir := range s.musicDirs { + err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error { + return s.watchCallback(dir, absPath, d, err) + }) + if err != nil { + log.Printf("error watching directory tree: %v\n", err) + } + } + + scanList := map[string]struct{}{} + for { + select { + case <-t.C: + if !s.StartScanning() { + scanList = map[string]struct{}{} + break + } + for dirName := range scanList { + c := &Context{ + errs: &multierr.Err{}, + seenTracks: map[int]struct{}{}, + seenAlbums: map[int]struct{}{}, + isFull: false, + } + musicDirName := s.watchMap[dirName] + if musicDirName == "" { + musicDirName = s.watchMap[filepath.Dir(dirName)] + } + err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error { + return s.watchCallback(musicDirName, absPath, d, err) + }) + if err != nil { + log.Printf("error watching directory tree: %v\n", err) + } + err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error { + return s.scanCallback(c, musicDirName, absPath, d, err) + }) + if err != nil { + log.Printf("error walking: %v", err) + } + + } + scanList = nil + s.StopScanning() + case event := <-s.watcher.Events: + var dirName string + if event.Op&(fsnotify.Create|fsnotify.Write) == 0 { + break + } + if len(scanList) == 0 { + t.Reset(10 * time.Second) + } + fileInfo, err := os.Stat(event.Name) + if err != nil && fileInfo.IsDir() { + dirName = event.Name + } else { + dirName = filepath.Dir(event.Name) + } + scanList[dirName] = struct{}{} + case err = <-s.watcher.Errors: + log.Printf("error from watcher: %v\n", err) + case <-s.watchDone: + return nil + } + } +} + +func (s *Scanner) CancelWatch() { + s.watchDone <- true +} + +func (s *Scanner) watchCallback(dir string, absPath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + switch d.Type() { + case os.ModeDir: + case os.ModeSymlink: + eval, _ := filepath.EvalSymlinks(absPath) + return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error { + subAbs = strings.Replace(subAbs, eval, absPath, 1) + return s.watchCallback(dir, subAbs, d, err) + }) + default: + return nil + } + + if s.watchMap[absPath] == "" { + s.watchMap[absPath] = dir + err = s.watcher.Add(absPath) + } + return err +} + func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirEntry, err error) error { if err != nil { c.errs.Add(err) diff --git a/server/server.go b/server/server.go index 3d859c48..dd716922 100644 --- a/server/server.go +++ b/server/server.go @@ -296,6 +296,7 @@ func (s *Server) StartHTTP(listenAddr string, tlsCert string, tlsKey string) (Fu Addr: listenAddr, Handler: s.router, ReadTimeout: 5 * time.Second, + ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 80 * time.Second, IdleTimeout: 60 * time.Second, } @@ -338,6 +339,16 @@ func (s *Server) StartScanTicker(dur time.Duration) (FuncExecute, FuncInterrupt) } } +func (s *Server) StartScanWatcher() (FuncExecute, FuncInterrupt) { + return func() error { + log.Printf("starting job 'scan watcher'\n") + return s.scanner.ExecuteWatch() + }, func(_ error) { + // stop job + s.scanner.CancelWatch() + } +} + func (s *Server) StartJukebox() (FuncExecute, FuncInterrupt) { return func() error { log.Printf("starting job 'jukebox'\n")