feat(scanner): add option to use fsnotify based scan watcher (#232)

* First cut at fsnotify support. Tested on Linux.

* Fixed bug in logging.

* Fixed lint issues.

* Added new scan watcher option to README.md

* Inverted conditional and dedented following code as per PR discussion.

* Changed command line switch and error return on ExecuteWatch() as per GH comments.

* Scan watcher: Removed scan at first start. Modified watcher to set a 10 second timer and then process in bulk.

Co-authored-by: Brian Doherty <brian@hplaptop.dohertyfamily.me>

README.md

@@ -161,6 +161,7 @@ view the admin UI at http://localhost:4747
 | `GONIC_TLS_KEY`         | `-tls-key`         | **optional** path to a TLS key (enables HTTPS listening)                                                    |
 | `GONIC_PROXY_PREFIX`    | `-proxy-prefix`    | **optional** url path prefix to use if behind reverse proxy. eg `/gonic` (see example configs below)        |
 | `GONIC_SCAN_INTERVAL`   | `-scan-interval`   | **optional** interval (in minutes) to check for new music (automatic scanning disabled if omitted)          |
+| `GONIC_SCAN_WATCHER `   | `-scan-watcher-enabled` | **optional** whether to watch file system for new music and rescan         |
 | `GONIC_JUKEBOX_ENABLED` | `-jukebox-enabled` | **optional** whether the subsonic [jukebox api](https://airsonic.github.io/docs/jukebox/) should be enabled |
 | `GONIC_GENRE_SPLIT`     | `-genre-split`     | **optional** a string or character to split genre tags on for multi-genre support (eg. `;`)                 |
 

cmd/gonic/gonic.go

@@ -36,6 +36,7 @@ func main() {
 	confCachePath := set.String("cache-path", "", "path to cache")
 	confDBPath := set.String("db-path", "gonic.db", "path to database (optional)")
 	confScanInterval := set.Int("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)")
+	confScanWatcher := set.Bool("scan-watcher-enabled", false, "whether to watch file system for new music and rescan (optional)")
 	confJukeboxEnabled := set.Bool("jukebox-enabled", false, "whether the subsonic jukebox api should be enabled (optional)")
 	confProxyPrefix := set.String("proxy-prefix", "", "url path prefix to use if behind proxy. eg '/gonic' (optional)")
 	confGenreSplit := set.String("genre-split", "\n", "character or string to split genre tag data on (optional)")
@@ -134,6 +135,9 @@ func main() {
 		tickerDur := time.Duration(*confScanInterval) * time.Minute
 		g.Add(server.StartScanTicker(tickerDur))
 	}
+	if *confScanWatcher {
+		g.Add(server.StartScanWatcher())
+	}
 	if *confJukeboxEnabled {
 		g.Add(server.StartJukebox())
 	}

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/faiface/beep v1.1.0
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

go.sum

@@ -35,6 +35,8 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DP
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/faiface/beep v1.1.0 h1:A2gWP6xf5Rh7RG/p9/VAW2jRSDEGQm5sbOb38sf5d4c=
 github.com/faiface/beep v1.1.0/go.mod h1:6I8p6kK2q4opL/eWb+kAkk38ehnTunWeToJB+s51sT4=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
@@ -258,6 +260,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d h1:Bm7BNOQt2Qv7ZqysjeLjgCBanX+88Z/OtdvsrEv1Djc=
 golang.org/x/sys v0.0.0-20220207234003-57398862261d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d h1:Zu/JngovGLVi6t2J3nmAf3AoTDwuzw85YZ3b9o4yU7s=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

scanner/scanner.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/jinzhu/gorm"
 	"github.com/rainycape/unidecode"
 
@@ -34,6 +35,9 @@ type Scanner struct {
 	genreSplit string
 	tagger     tags.Reader
 	scanning   *int32
+	watcher    *fsnotify.Watcher
+	watchMap   map[string]string // maps watched dirs back to root music dir
+	watchDone  chan bool
 }
 
 func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
@@ -43,23 +47,30 @@ func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *
 		genreSplit: genreSplit,
 		tagger:     tagger,
 		scanning:   new(int32),
+		watchMap:   make(map[string]string),
+		watchDone:  make(chan bool),
 	}
 }
 
 func (s *Scanner) IsScanning() bool {
 	return atomic.LoadInt32(s.scanning) == 1
 }
+func (s *Scanner) StartScanning() bool {
+	return atomic.CompareAndSwapInt32(s.scanning, 0, 1)
+}
+func (s *Scanner) StopScanning() {
+	defer atomic.StoreInt32(s.scanning, 0)
+}
 
 type ScanOptions struct {
 	IsFull bool
 }
 
 func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
-	if s.IsScanning() {
+	if !s.StartScanning() {
 		return nil, ErrAlreadyScanning
 	}
-	atomic.StoreInt32(s.scanning, 1)
-	defer atomic.StoreInt32(s.scanning, 0)
+	defer s.StopScanning()
 
 	start := time.Now()
 	c := &Context{
@@ -108,6 +119,115 @@ func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
 	return c, nil
 }
 
+func (s *Scanner) ExecuteWatch() error {
+	var err error
+	s.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		log.Printf("error creating watcher: %v\n", err)
+		return err
+	}
+	defer s.watcher.Close()
+
+	t := time.NewTimer(10 * time.Second)
+	if !t.Stop() {
+		<-t.C
+	}
+
+	for _, dir := range s.musicDirs {
+		err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
+			return s.watchCallback(dir, absPath, d, err)
+		})
+		if err != nil {
+			log.Printf("error watching directory tree: %v\n", err)
+		}
+	}
+
+	scanList := map[string]struct{}{}
+	for {
+		select {
+		case <-t.C:
+			if !s.StartScanning() {
+				scanList = map[string]struct{}{}
+				break
+			}
+			for dirName := range scanList {
+				c := &Context{
+					errs:       &multierr.Err{},
+					seenTracks: map[int]struct{}{},
+					seenAlbums: map[int]struct{}{},
+					isFull:     false,
+				}
+				musicDirName := s.watchMap[dirName]
+				if musicDirName == "" {
+					musicDirName = s.watchMap[filepath.Dir(dirName)]
+				}
+				err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error {
+					return s.watchCallback(musicDirName, absPath, d, err)
+				})
+				if err != nil {
+					log.Printf("error watching directory tree: %v\n", err)
+				}
+				err = filepath.WalkDir(dirName, func(absPath string, d fs.DirEntry, err error) error {
+					return s.scanCallback(c, musicDirName, absPath, d, err)
+				})
+				if err != nil {
+					log.Printf("error walking: %v", err)
+				}
+
+			}
+			scanList = nil
+			s.StopScanning()
+		case event := <-s.watcher.Events:
+			var dirName string
+			if event.Op&(fsnotify.Create|fsnotify.Write) == 0 {
+				break
+			}
+			if len(scanList) == 0 {
+				t.Reset(10 * time.Second)
+			}
+			fileInfo, err := os.Stat(event.Name)
+			if err != nil && fileInfo.IsDir() {
+				dirName = event.Name
+			} else {
+				dirName = filepath.Dir(event.Name)
+			}
+			scanList[dirName] = struct{}{}
+		case err = <-s.watcher.Errors:
+			log.Printf("error from watcher: %v\n", err)
+		case <-s.watchDone:
+			return nil
+		}
+	}
+}
+
+func (s *Scanner) CancelWatch() {
+	s.watchDone <- true
+}
+
+func (s *Scanner) watchCallback(dir string, absPath string, d fs.DirEntry, err error) error {
+	if err != nil {
+		return err
+	}
+
+	switch d.Type() {
+	case os.ModeDir:
+	case os.ModeSymlink:
+		eval, _ := filepath.EvalSymlinks(absPath)
+		return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
+			subAbs = strings.Replace(subAbs, eval, absPath, 1)
+			return s.watchCallback(dir, subAbs, d, err)
+		})
+	default:
+		return nil
+	}
+
+	if s.watchMap[absPath] == "" {
+		s.watchMap[absPath] = dir
+		err = s.watcher.Add(absPath)
+	}
+	return err
+}
+
 func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirEntry, err error) error {
 	if err != nil {
 		c.errs.Add(err)

server/server.go

@@ -296,6 +296,7 @@ func (s *Server) StartHTTP(listenAddr string, tlsCert string, tlsKey string) (Fu
 		Addr:         listenAddr,
 		Handler:      s.router,
 		ReadTimeout:  5 * time.Second,
+		ReadHeaderTimeout:  5 * time.Second,
 		WriteTimeout: 80 * time.Second,
 		IdleTimeout:  60 * time.Second,
 	}
@@ -338,6 +339,16 @@ func (s *Server) StartScanTicker(dur time.Duration) (FuncExecute, FuncInterrupt)
 		}
 }
 
+func (s *Server) StartScanWatcher() (FuncExecute, FuncInterrupt) {
+	return func() error {
+			log.Printf("starting job 'scan watcher'\n")
+			return s.scanner.ExecuteWatch()
+		}, func(_ error) {
+			// stop job
+			s.scanner.CancelWatch()
+		}
+}
+
 func (s *Server) StartJukebox() (FuncExecute, FuncInterrupt) {
 	return func() error {
 			log.Printf("starting job 'jukebox'\n")