WIP: Add folders to DB

db/migration/20231212210020_add_library_table.go

@@ -17,9 +17,10 @@ func upAddLibraryTable(ctx context.Context, tx *sql.Tx) error {
 	_, err := tx.ExecContext(ctx, `
 		create table library (
 			id integer primary key autoincrement,
-			name text not null unique,
-			path text not null unique,
-			remote_path text null default '',
+			name varchar not null unique,
+			path varchar not null unique,
+			remote_path varchar null default '',
+			extractor varchar null default 'taglib',
 			last_scan_at datetime not null default '0000-00-00 00:00:00',
 			updated_at datetime not null default current_timestamp,
 			created_at datetime not null default current_timestamp
@@ -29,9 +30,9 @@ func upAddLibraryTable(ctx context.Context, tx *sql.Tx) error {
 	}
 
 	_, err = tx.ExecContext(ctx, fmt.Sprintf(`
-		insert into library(id, name, path, last_scan_at) values(1, 'Music Library', '%s', current_timestamp);
+		insert into library(id, name, path, extractor, last_scan_at) values(1, 'Music Library', '%s', '%s', current_timestamp);
 		delete from property where id like 'LastScan-%%';
-`, conf.Server.MusicFolder))
+`, conf.Server.MusicFolder, conf.Server.Scanner.Extractor))
 	if err != nil {
 		return err
 	}

db/migration/20231219003407_add_folder_table.go

@@ -0,0 +1,38 @@
+package migrations
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigrationContext(upAddFolderTable, downAddFolderTable)
+}
+
+func upAddFolderTable(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, `
+create table if not exists folder(
+	id varchar not null
+		primary key,
+	library_id integer not null
+	    		references library (id)
+	    		 	on delete cascade,
+	path varchar default '' not null,
+	name varchar default '' not null,
+	updated_at timestamp default current_timestamp not null,
+	created_at timestamp default current_timestamp not null,
+	parent_id varchar default null
+		references folder (id)
+		 	on delete cascade
+);
+`)
+
+	return err
+}
+
+func downAddFolderTable(ctx context.Context, tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

model/datastore.go

@@ -21,6 +21,7 @@ type ResourceRepository interface {
 
 type DataStore interface {
 	Library(ctx context.Context) LibraryRepository
+	Folder(ctx context.Context) FolderRepository
 	Album(ctx context.Context) AlbumRepository
 	Artist(ctx context.Context) ArtistRepository
 	MediaFile(ctx context.Context) MediaFileRepository

model/folder.go

@@ -0,0 +1,42 @@
+package model
+
+import (
+	"crypto/md5"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type Folder struct {
+	ID        string
+	LibraryID int
+	Path      string
+	Name      string
+	ParentID  string
+	UpdateAt  time.Time
+	CreatedAt time.Time
+}
+
+func FolderID(lib Library, path string) string {
+	path = strings.TrimPrefix(path, lib.Path)
+	key := fmt.Sprintf("%d:%s", lib.ID, path)
+	return fmt.Sprintf("%x", md5.Sum([]byte(key)))
+}
+func NewFolder(lib Library, path string) *Folder {
+	id := FolderID(lib, path)
+	dir, name := filepath.Split(path)
+	return &Folder{
+		ID:   id,
+		Path: dir,
+		Name: name,
+	}
+}
+
+type FolderRepository interface {
+	Get(lib Library, path string) (*Folder, error)
+	GetAll(lib Library) ([]Folder, error)
+	GetLastUpdates(lib Library) (map[string]time.Time, error)
+	Put(lib Library, path string) error
+	Touch(lib Library, path string, t time.Time) error
+}

model/library.go

@@ -11,6 +11,7 @@ type Library struct {
 	Name       string
 	Path       string
 	RemotePath string
+	Extractor  string
 	LastScanAt time.Time
 	UpdatedAt  time.Time
 	CreatedAt  time.Time

persistence/folder_repository.go

@@ -0,0 +1,69 @@
+package persistence
+
+import (
+	"context"
+	"time"
+
+	. "github.com/Masterminds/squirrel"
+	"github.com/navidrome/navidrome/model"
+	"github.com/pocketbase/dbx"
+)
+
+type folderRepository struct {
+	sqlRepository
+}
+
+func newFolderRepository(ctx context.Context, db dbx.Builder) model.FolderRepository {
+	r := &folderRepository{}
+	r.ctx = ctx
+	r.db = db
+	r.tableName = "folder"
+	return r
+}
+
+func (r folderRepository) Get(lib model.Library, path string) (*model.Folder, error) {
+	id := model.NewFolder(lib, path).ID
+	sq := r.newSelect().Where(Eq{"id": id})
+	var res model.Folder
+	err := r.queryOne(sq, res)
+	return &res, err
+}
+
+func (r folderRepository) GetAll(lib model.Library) ([]model.Folder, error) {
+	sq := r.newSelect().Columns("*").Where(Eq{"library_id": lib.ID})
+	var res []model.Folder
+	err := r.queryAll(sq, &res)
+	return res, err
+}
+
+func (r folderRepository) GetLastUpdates(lib model.Library) (map[string]time.Time, error) {
+	sq := r.newSelect().Columns("id", "updated_at").Where(Eq{"library_id": lib.ID})
+	var res []struct {
+		ID        string
+		UpdatedAt time.Time
+	}
+	err := r.queryAll(sq, &res)
+	if err != nil {
+		return nil, err
+	}
+	m := make(map[string]time.Time, len(res))
+	for _, f := range res {
+		m[f.ID] = f.UpdatedAt
+	}
+	return m, nil
+}
+
+func (r folderRepository) Put(lib model.Library, path string) error {
+	folder := model.NewFolder(lib, path)
+	_, err := r.put(folder.ID, folder)
+	return err
+}
+
+func (r folderRepository) Touch(lib model.Library, path string, t time.Time) error {
+	id := model.FolderID(lib, path)
+	sq := Update(r.tableName).Set("updated_at", t).Where(Eq{"id": id})
+	_, err := r.executeSQL(sq)
+	return err
+}
+
+var _ model.FolderRepository = (*folderRepository)(nil)

persistence/library_repository.go

@@ -52,7 +52,7 @@ const hardCodedMusicFolderID = 1
 
 func (r *libraryRepository) StoreMusicFolder() error {
 	sq := Update(r.tableName).Set("path", conf.Server.MusicFolder).Set("updated_at", time.Now()).
-		Where(Eq{"id": hardCodedMusicFolderID})
+		Set("extractor", conf.Server.Scanner.Extractor).Where(Eq{"id": hardCodedMusicFolderID})
 	_, err := r.executeSQL(sq)
 	return err
 }

persistence/persistence.go

@@ -35,6 +35,10 @@ func (s *SQLStore) Library(ctx context.Context) model.LibraryRepository {
 	return NewLibraryRepository(ctx, s.getDBXBuilder())
 }
 
+func (s *SQLStore) Folder(ctx context.Context) model.FolderRepository {
+	return newFolderRepository(ctx, s.getDBXBuilder())
+}
+
 func (s *SQLStore) Genre(ctx context.Context) model.GenreRepository {
 	return NewGenreRepository(ctx, s.getDBXBuilder())
 }

scanner2/folder.go

@@ -24,6 +24,10 @@ type folderEntry struct {
 	audioFilesCount uint32
 }
 
+func (f *folderEntry) IsExpired() bool {
+	return f.scanCtx.getLastUpdatedInDB(f.path).Before(f.modTime)
+}
+
 func loadDir(ctx context.Context, scanCtx *scanContext, dirPath string, d fastwalk.DirEntry) (folder *folderEntry, children []string, err error) {
 	folder = &folderEntry{DirEntry: d, scanCtx: scanCtx, path: dirPath}
 

scanner2/scan_context.go

@@ -1,19 +1,43 @@
 package scanner2
 
 import (
+	"context"
+	"fmt"
+	"sync"
 	"time"
 
 	"github.com/navidrome/navidrome/model"
 )
 
 type scanContext struct {
-	lib       model.Library
-	startTime time.Time
+	lib         model.Library
+	ds          model.DataStore
+	startTime   time.Time
+	lastUpdates map[string]time.Time
+	lock        sync.RWMutex
 }
 
-func newScannerContext(lib model.Library) *scanContext {
+func newScannerContext(ctx context.Context, ds model.DataStore, lib model.Library) (*scanContext, error) {
+	lastUpdates, err := ds.Folder(ctx).GetLastUpdates(lib)
+	if err != nil {
+		return nil, fmt.Errorf("error getting last updates: %w", err)
+	}
 	return &scanContext{
-		lib:       lib,
-		startTime: time.Now(),
+		lib:         lib,
+		ds:          ds,
+		startTime:   time.Now(),
+		lastUpdates: lastUpdates,
+	}, nil
+}
+
+func (s *scanContext) getLastUpdatedInDB(path string) time.Time {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	id := model.FolderID(s.lib, path)
+	t, ok := s.lastUpdates[id]
+	if !ok {
+		return time.Time{}
 	}
+	return t
 }

scanner2/scanner2.go

@@ -32,28 +32,44 @@ func (s *scanner2) RescanAll(requestCtx context.Context, fullRescan bool) error
 
 	startTime := time.Now()
 	log.Info(ctx, "Scanner: Starting scan", "fullRescan", fullRescan, "numLibraries", len(libs))
-	scanCtxChan := createScanContexts(ctx, libs)
+
+	scanCtxChan := createScanContexts(ctx, s.ds, libs)
 	folderChan, folderErrChan := walkDirEntries(ctx, scanCtxChan)
-	logErrChan := pl.Sink(ctx, 4, folderChan, func(ctx context.Context, folder *folderEntry) error {
+	changedFolderChan, changedFolderErrChan := pl.Filter(ctx, 4, folderChan, filterOutUpdated(fullRescan))
+
+	// TODO Next: load tags from all files in folder
+
+	logErrChan := pl.Sink(ctx, 4, changedFolderChan, func(ctx context.Context, folder *folderEntry) error {
 		log.Debug(ctx, "Scanner: Found folder", "folder", folder.Name(), "_path", folder.path, "audioCount", folder.audioFilesCount, "images", folder.images, "hasPlaylist", folder.hasPlaylists)
 		return nil
 	})
 
 	// Wait for pipeline to end, return first error found
-	for err := range pl.Merge(ctx, folderErrChan, logErrChan) {
+	for err := range pl.Merge(ctx, folderErrChan, logErrChan, changedFolderErrChan) {
 		return err
 	}
 
-	log.Info(ctx, "Scanner: Scan finished", "duration", time.Since(startTime))
+	log.Info(ctx, "Scanner: Finished scanning all libraries", "duration", time.Since(startTime))
 	return nil
 }
 
-func createScanContexts(ctx context.Context, libs []model.Library) chan *scanContext {
+func filterOutUpdated(fullScan bool) func(ctx context.Context, entry *folderEntry) (bool, error) {
+	return func(ctx context.Context, entry *folderEntry) (bool, error) {
+		return fullScan || entry.IsExpired(), nil
+	}
+}
+
+func createScanContexts(ctx context.Context, ds model.DataStore, libs []model.Library) chan *scanContext {
 	outputChannel := make(chan *scanContext, len(libs))
 	go func() {
 		defer close(outputChannel)
 		for _, lib := range libs {
-			outputChannel <- newScannerContext(lib)
+			scanCtx, err := newScannerContext(ctx, ds, lib)
+			if err != nil {
+				log.Error(ctx, "Scanner: Error creating scan context", "lib", lib.Name, err)
+				continue
+			}
+			outputChannel <- scanCtx
 		}
 	}()
 	return outputChannel

tests/mock_persistence.go

@@ -47,6 +47,10 @@ func (db *MockDataStore) Library(context.Context) model.LibraryRepository {
 	return struct{ model.LibraryRepository }{}
 }
 
+func (db *MockDataStore) Folder(context.Context) model.FolderRepository {
+	return struct{ model.FolderRepository }{}
+}
+
 func (db *MockDataStore) Genre(context.Context) model.GenreRepository {
 	if db.MockedGenre == nil {
 		db.MockedGenre = &MockedGenreRepo{}

utils/pl/pipelines.go

@@ -174,3 +174,32 @@ func FromSlice[T any](ctx context.Context, in []T) <-chan T {
 	close(output)
 	return output
 }
+
+func Filter[T any](ctx context.Context, maxWorkers int, inputChan chan T, f func(context.Context, T) (bool, error)) (chan T, chan error) {
+	outputChan := make(chan T)
+	errorChan := make(chan error)
+	go func() {
+		defer close(outputChan)
+		defer close(errorChan)
+
+		errChan := Sink(ctx, maxWorkers, inputChan, func(ctx context.Context, item T) error {
+			ok, err := f(ctx, item)
+			if err != nil {
+				return err
+			}
+			if ok {
+				outputChan <- item
+			}
+			return nil
+		})
+
+		// Wait for pipeline to end, and forward any errors
+		for err := range ReadOrDone(ctx, errChan) {
+			select {
+			case errorChan <- err:
+			default:
+			}
+		}
+	}()
+	return outputChan, errorChan
+}