Skip to content

Commit

Permalink
create frontend
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmanibus
rmanibus committed Jan 14, 2024
1 parent a9a418d commit bbae8cb
Show file tree
Hide file tree
Showing 34 changed files with 1,725 additions and 732 deletions.
185 changes: 98 additions & 87 deletions cmd/restic/cmd_backup.go
View file
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strconv"
Expand All @@ -20,6 +21,7 @@ import (
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/frontend"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
Expand Down Expand Up @@ -164,25 +166,22 @@ func init() {

// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(targetFS fs.FS, items []string) (result []string, err error) {
return items, nil
/*
for _, item := range items {
_, err := targetFS.Lstat(item)
if errors.Is(err, os.ErrNotExist) {
Warnf("%v does not exist, skipping\n", item)
continue
}
func filterExisting(items []restic.LazyFileMetadata) (result []restic.LazyFileMetadata, err error) {
for _, item := range items {

result = append(result, item)
if !item.Exist() {
Warnf("%v does not exist, skipping\n", item)
continue
}

if len(result) == 0 {
return nil, errors.Fatal("all target directories/files do not exist")
}
result = append(result, item)
}

return
*/
if len(result) == 0 {
return nil, errors.Fatal("all target directories/files do not exist")
}

return
}

// readLines reads all lines from the named file and returns them as a
Expand Down Expand Up @@ -347,7 +346,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (

// collectRejectFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path and file info
func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
func collectRejectFuncs(opts BackupOptions, targets []restic.LazyFileMetadata) (fs []RejectFunc, err error) {
// allowed devices
if opts.ExcludeOtherFS && !opts.Stdin {
f, err := rejectByDevice(targets)
Expand All @@ -369,7 +368,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc,
}

// collectTargets returns a list of target files/dirs from several sources.
func collectTargets(targetFS fs.FS, opts BackupOptions, args []string) (targets []string, err error) {
func collectTargets(f frontend.Frontend, opts BackupOptions, args []string) (targets []restic.LazyFileMetadata, err error) {
if opts.Stdin || opts.StdinCommand {
return nil, nil
}
Expand All @@ -395,7 +394,7 @@ func collectTargets(targetFS fs.FS, opts BackupOptions, args []string) (targets
if len(expanded) == 0 {
Warnf("pattern %q does not match any files, skipping\n", line)
}
targets = append(targets, expanded...)
targets = append(targets, f.Prepare(expanded...)...)
}
}

Expand All @@ -408,7 +407,7 @@ func collectTargets(targetFS fs.FS, opts BackupOptions, args []string) (targets
if line == "" {
continue
}
targets = append(targets, line)
targets = append(targets, f.Prepare(line)...)
}
}

Expand All @@ -417,17 +416,17 @@ func collectTargets(targetFS fs.FS, opts BackupOptions, args []string) (targets
if err != nil {
return nil, err
}
targets = append(targets, fromfile...)
targets = append(targets, f.Prepare(fromfile...)...)
}

// Merge args into files-from so we can reuse the normal args checks
// and have the ability to use both files-from and args at the same time.
targets = append(targets, args...)
targets = append(targets, f.Prepare(args...)...)
if len(targets) == 0 && !opts.Stdin {
return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
}

targets, err = filterExisting(targetFS, targets)
targets, err = filterExisting(targets)
if err != nil {
return nil, err
}
Expand All @@ -437,7 +436,7 @@ func collectTargets(targetFS fs.FS, opts BackupOptions, args []string) (targets

// parent returns the ID of the parent snapshot. If there is none, nil is
// returned.
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []restic.LazyFileMetadata, timeStampLimit time.Time) (*restic.Snapshot, error) {
if opts.Force {
return nil, nil
}
Expand All @@ -451,7 +450,11 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
f.Hosts = []string{opts.Host}
}
if opts.GroupBy.Path {
f.Paths = targets
paths := make([]string, len(targets))
for i, target := range targets {
paths[i] = target.Path()
}
f.Paths = paths
}
if opts.GroupBy.Tag {
f.Tags = []restic.TagList{opts.Tags.Flatten()}
Expand All @@ -465,66 +468,77 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
return sn, err
}

func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
}

var targetFS fs.FS = fs.NewGoogleDrive()

//debug.Log("%v, %v", targets, err)

/*
if runtime.GOOS == "windows" && opts.UseFsSnapshot {
if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
return err
}
func newLocalFrontend(ctx context.Context, opts BackupOptions, gopts GlobalOptions, timeStamp time.Time, progressPrinter backup.ProgressPrinter, progressReporter *backup.Progress, args []string) (frontend.Frontend, []restic.LazyFileMetadata, error) {
var err error = nil
var targetFS fs.FS = fs.Local{}
var targets []string

errorHandler := func(item string, err error) error {
return progressReporter.Error(item, err)
}
messageHandler := func(msg string, args ...interface{}) {
if !gopts.JSON {
progressPrinter.P(msg, args...)
}
}
if runtime.GOOS == "windows" && opts.UseFsSnapshot {
if err := fs.HasSufficientPrivilegesForVSS(); err != nil {
return nil, nil, err
}

localVss := fs.NewLocalVss(errorHandler, messageHandler)
defer localVss.DeleteSnapshots()
targetFS = localVss
}
errorHandler := func(item string, err error) error {
return progressReporter.Error(item, err)
}

if opts.Stdin || opts.StdinCommand {
messageHandler := func(msg string, args ...interface{}) {
if !gopts.JSON {
progressPrinter.V("read data from stdin")
progressPrinter.P(msg, args...)
}
filename := path.Join("/", opts.StdinFilename)
var source io.ReadCloser = os.Stdin
if opts.StdinCommand {
source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
if err != nil {
return err
}
}
targetFS = &fs.Reader{
ModTime: timeStamp,
Name: filename,
Mode: 0644,
ReadCloser: source,
}
targets = []string{filename}
}

targets, err := collectTargets(targetFS, opts, args)
localVss := fs.NewLocalVss(errorHandler, messageHandler)
defer localVss.DeleteSnapshots()
targetFS = localVss
}

if err != nil {
return err
if opts.Stdin || opts.StdinCommand {
if !gopts.JSON {
progressPrinter.V("read data from stdin")
}
filename := path.Join("/", opts.StdinFilename)
var source io.ReadCloser = os.Stdin
if opts.StdinCommand {
source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
if err != nil {
return nil, nil, err
}
}
targetFS = &fs.Reader{
ModTime: timeStamp,
Name: filename,
Mode: 0644,
ReadCloser: source,
}
*/
targets = []string{filename}
}

f := &frontend.LocalFrontend{
FS: targetFS,
}

targets := []string{"root"}
if opts.IgnoreInode {
// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
// reliable either.
f.ChangeIgnoreFlags |= frontend.ChangeIgnoreCtime | frontend.ChangeIgnoreInode
}
if opts.IgnoreCtime {
f.ChangeIgnoreFlags |= frontend.ChangeIgnoreCtime
}

if targets == nil {
t, err := collectTargets(f, opts, args)
return f, t, err
}
return f, f.Prepare(targets...), nil
}

func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
}

timeStamp := time.Now()
if opts.TimeStamp != "" {
Expand Down Expand Up @@ -569,6 +583,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
}
}

f, targets, err := newLocalFrontend(ctx, opts, gopts, timeStamp, progressPrinter, progressReporter, args)

if err != nil {
return err
}

// rejectByNameFuncs collect functions that can reject items from the backup based on path only
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
if err != nil {
Expand Down Expand Up @@ -617,9 +637,9 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
return true
}

selectFilter := func(item string, fi os.FileInfo) bool {
selectFilter := func(fm restic.FileMetadata) bool {
for _, reject := range rejectFuncs {
if reject(item, fi) {
if reject(fm) {
return false
}
}
Expand All @@ -631,7 +651,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
defer cancel()

if !opts.NoScan {
sc := archiver.NewScanner(targetFS)
sc := archiver.NewScanner(f)
sc.SelectByName = selectByNameFilter
sc.Select = selectFilter
sc.Error = progressPrinter.ScannerError
Expand All @@ -643,7 +663,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
}

arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
arch := archiver.New(repo, f, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
arch.SelectByName = selectByNameFilter
arch.Select = selectFilter
arch.WithAtime = opts.WithAtime
Expand All @@ -662,15 +682,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
arch.StartFile = progressReporter.StartFile
arch.CompleteBlob = progressReporter.CompleteBlob

if opts.IgnoreInode {
// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
// reliable either.
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime | archiver.ChangeIgnoreInode
}
if opts.IgnoreCtime {
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
}

snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes,
Tags: opts.Tags.Flatten(),
Expand Down
8 changes: 6 additions & 2 deletions cmd/restic/cmd_backup_test.go
View file
Expand Up @@ -10,6 +10,8 @@ import (
"strings"
"testing"

"github.com/restic/restic/internal/frontend"
"github.com/restic/restic/internal/fs"
rtest "github.com/restic/restic/internal/test"
)

Expand Down Expand Up @@ -64,9 +66,11 @@ func TestCollectTargets(t *testing.T) {
FilesFromRaw: []string{f3.Name()},
}

targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
targets, err := collectTargets(&frontend.LocalFrontend{FS: fs.Local{}}, opts, []string{filepath.Join(dir, "cmdline arg")})
rtest.OK(t, err)
sort.Strings(targets)
sort.Slice(targets, func(a, b int) bool {
return targets[a].Name() < targets[b].Name()
})
rtest.Equals(t, expect, targets)
}

Expand Down
4 changes: 3 additions & 1 deletion cmd/restic/cmd_recover.go
View file
Expand Up @@ -6,6 +6,7 @@ import (
"time"

"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/frontend"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -159,7 +160,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
}

func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
f := frontend.LocalFrontend{}
sn, err := restic.NewSnapshot([]restic.FilePath{restic.FilePath(f.Prepare(name)[0])}, tags, hostname, time.Now())
if err != nil {
return errors.Fatalf("unable to save snapshot: %v", err)
}
Expand Down

0 comments on commit bbae8cb

Please sign in to comment.