From 296cc1bca2bd6095e91a634ace5603e436e21860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Wilczy=C5=84ski?= Date: Thu, 30 Sep 2021 19:19:18 +0900 Subject: [PATCH 001/220] lib/model: Limit the number of default hashers on Android (ref #2220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Like Windows and Mac, Android is also an interactive operating system. On top of that, it usually runs on much slower hardware than the other two. Because of that, it makes sense to limit the number of hashes used by default there too. Signed-off-by: Tomasz Wilczyński --- lib/model/model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/model/model.go b/lib/model/model.go index 2218bd8020d..c1b3ddaa810 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -2403,7 +2403,7 @@ func (m *model) numHashers(folder string) int { return folderCfg.Hashers } - if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" || runtime.GOOS == "android" { // Interactive operating systems; don't load the system too heavily by // default. return 1 From 807a6b10221ea0a01dfedab0ce26473f20266761 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Fri, 29 Oct 2021 20:20:46 +0200 Subject: [PATCH 002/220] lib/model: Optimize jobQueue performance and memory use (#8023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By truncating time.Time to an int64 nanosecond count, we lose the ability to precisely order timestamps before 1678 or after 2262, but we gain (linux/amd64, Go 1.17.1): name old time/op new time/op delta JobQueuePushPopDone10k-8 2.85ms ± 5% 2.29ms ± 2% -19.80% (p=0.000 n=20+18) JobQueueBump-8 34.0µs ± 1% 29.8µs ± 1% -12.35% (p=0.000 n=19+19) name old alloc/op new alloc/op delta JobQueuePushPopDone10k-8 2.56MB ± 0% 1.76MB ± 0% -31.31% (p=0.000 n=18+13) name old allocs/op new allocs/op delta JobQueuePushPopDone10k-8 23.0 ± 0% 23.0 ± 0% ~ (all equal) Results for BenchmarkJobQueueBump are with the fixed version, which no longer depends on b.N for the amount of work performed. rand.Rand.Intn is cheap at ~10ns per iteration. --- lib/model/queue.go | 7 ++++--- lib/model/queue_test.go | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/model/queue.go b/lib/model/queue.go index 28070a9a47f..16ac83c9434 100644 --- a/lib/model/queue.go +++ b/lib/model/queue.go @@ -23,7 +23,7 @@ type jobQueue struct { type jobQueueEntry struct { name string size int64 - modified time.Time + modified int64 } func newJobQueue() *jobQueue { @@ -34,7 +34,8 @@ func newJobQueue() *jobQueue { func (q *jobQueue) Push(file string, size int64, modified time.Time) { q.mut.Lock() - q.queued = append(q.queued, jobQueueEntry{file, size, modified}) + // The range of UnixNano covers a range of reasonable timestamps. + q.queued = append(q.queued, jobQueueEntry{file, size, modified.UnixNano()}) q.mut.Unlock() } @@ -191,5 +192,5 @@ func (q smallestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] } type oldestFirst []jobQueueEntry func (q oldestFirst) Len() int { return len(q) } -func (q oldestFirst) Less(a, b int) bool { return q[a].modified.Before(q[b].modified) } +func (q oldestFirst) Less(a, b int) bool { return q[a].modified < q[b].modified } func (q oldestFirst) Swap(a, b int) { q[a], q[b] = q[b], q[a] } diff --git a/lib/model/queue_test.go b/lib/model/queue_test.go index 677c056dd72..9f49a683b2e 100644 --- a/lib/model/queue_test.go +++ b/lib/model/queue_test.go @@ -8,6 +8,7 @@ package model import ( "fmt" + "math/rand" "testing" "time" @@ -251,16 +252,19 @@ func TestSortByAge(t *testing.T) { } func BenchmarkJobQueueBump(b *testing.B) { - files := genFiles(b.N) + files := genFiles(10000) q := newJobQueue() for _, f := range files { q.Push(f.Name, 0, time.Time{}) } + rng := rand.New(rand.NewSource(int64(b.N))) + b.ResetTimer() for i := 0; i < b.N; i++ { - q.BringToFront(files[i].Name) + r := rng.Intn(len(files)) + q.BringToFront(files[r].Name) } } From 28ff033da65292868bd8e8f82d3083da2c338ff2 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Fri, 29 Oct 2021 20:21:50 +0200 Subject: [PATCH 003/220] cmd/syncthing/cli: indexDumpSize doesn't need a heap (#8024) --- cmd/syncthing/cli/index_dumpsize.go | 46 ++++++++++------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/cmd/syncthing/cli/index_dumpsize.go b/cmd/syncthing/cli/index_dumpsize.go index 35e095cd80a..79c399d9838 100644 --- a/cmd/syncthing/cli/index_dumpsize.go +++ b/cmd/syncthing/cli/index_dumpsize.go @@ -7,53 +7,35 @@ package cli import ( - "container/heap" "encoding/binary" "fmt" + "sort" "github.com/urfave/cli" "github.com/syncthing/syncthing/lib/db" ) -type SizedElement struct { - key string - size int -} - -type ElementHeap []SizedElement - -func (h ElementHeap) Len() int { return len(h) } -func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size } -func (h ElementHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } - -func (h *ElementHeap) Push(x interface{}) { - *h = append(*h, x.(SizedElement)) -} - -func (h *ElementHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} - func indexDumpSize(*cli.Context) error { + type sizedElement struct { + key string + size int + } + ldb, err := getDB() if err != nil { return err } - h := &ElementHeap{} - heap.Init(h) - it, err := ldb.NewPrefixIterator(nil) if err != nil { return err } - var ele SizedElement + + var elems []sizedElement for it.Next() { + var ele sizedElement + key := it.Key() switch key[0] { case db.KeyTypeDevice: @@ -94,11 +76,13 @@ func indexDumpSize(*cli.Context) error { ele.key = fmt.Sprintf("UNKNOWN:%x", key) } ele.size = len(it.Value()) - heap.Push(h, ele) + elems = append(elems, ele) } - for h.Len() > 0 { - ele = heap.Pop(h).(SizedElement) + sort.Slice(elems, func(i, j int) bool { + return elems[i].size > elems[j].size + }) + for _, ele := range elems { fmt.Println(ele.key, ele.size) } From 1c2e96a5ca470773bdfd176a9e84e54b168e4d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Colomb?= Date: Fri, 29 Oct 2021 20:23:41 +0200 Subject: [PATCH 004/220] gui: Display identicons for discovered device IDs. (#8022) --- gui/default/syncthing/device/editDeviceModalView.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gui/default/syncthing/device/editDeviceModalView.html b/gui/default/syncthing/device/editDeviceModalView.html index 38fb995c41b..0b772c4373f 100644 --- a/gui/default/syncthing/device/editDeviceModalView.html +++ b/gui/default/syncthing/device/editDeviceModalView.html @@ -25,7 +25,11 @@

You can also select one of these nearby devices:

From 41bfb7a3302ea7e751fb40a6b8b5a021a0634f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Colomb?= Date: Thu, 4 Nov 2021 08:42:55 +0100 Subject: [PATCH 005/220] Normalize CLI options to always use two dashes. (#8037) Consistently use double dashes and fix typos -conf, -data-dir and -verify. Applies also to tests running the syncthing binary for consistency. * Fix mismatched option name --conf in cli subcommand. According to the source code comments, the cli option flags should mirror those from the serve subcommand where applicable. That one is actually called --config though. * cli: Fix help text option placeholders. The urfave/cli package uses the Value field of StringFlag to provide a default value, not to name the placeholder. That is instead done with backticks around some part of the Usage field. * cli: Add missing --data flag in subcommand help text. The urfave/cli based option parsing uses a fake flags collection to generate help texts matching the used global options. But the --data option was omitted from it, although it is definitely required when using --config as well. Note that it cannot just be ignored, as some debug stuff actually uses the DB: syncthing cli --data=/bar --config=/foo debug index dump --- cmd/syncthing/cli/main.go | 18 +++++++++--------- cmd/syncthing/cmdutil/util.go | 4 ++-- cmd/syncthing/decrypt/decrypt.go | 2 +- cmd/syncthing/main.go | 4 ++-- lib/locations/locations.go | 6 +++--- lib/syncthing/utils.go | 2 +- test/cli_test.go | 8 ++++---- test/override_test.go | 4 ++-- test/util.go | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cmd/syncthing/cli/main.go b/cmd/syncthing/cli/main.go index f1ef9e4a4d2..8e08b92ef5a 100644 --- a/cmd/syncthing/cli/main.go +++ b/cmd/syncthing/cli/main.go @@ -61,23 +61,23 @@ func Run() error { fakeFlags := []cli.Flag{ cli.StringFlag{ Name: "gui-address", - Value: "URL", - Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")", + Usage: "Override GUI address to `URL` (e.g. \"http://192.0.2.42:8443\")", }, cli.StringFlag{ Name: "gui-apikey", - Value: "API-KEY", - Usage: "Override GUI API key", + Usage: "Override GUI API key to `API-KEY`", }, cli.StringFlag{ Name: "home", - Value: "PATH", - Usage: "Set configuration and data directory", + Usage: "Set configuration and data directory to `PATH`", }, cli.StringFlag{ - Name: "conf", - Value: "PATH", - Usage: "Set configuration directory (config and keys)", + Name: "config", + Usage: "Set configuration directory (config and keys) to `PATH`", + }, + cli.StringFlag{ + Name: "data", + Usage: "Set data directory (database and logs) to `PATH`", }, } diff --git a/cmd/syncthing/cmdutil/util.go b/cmd/syncthing/cmdutil/util.go index 7c15f299513..6a44456b095 100644 --- a/cmd/syncthing/cmdutil/util.go +++ b/cmd/syncthing/cmdutil/util.go @@ -18,9 +18,9 @@ func SetConfigDataLocationsFromFlags(homeDir, confDir, dataDir string) error { dataSet := dataDir != "" switch { case dataSet != confSet: - return errors.New("either both or none of -conf and -data must be given, use -home to set both at once") + return errors.New("either both or none of --config and --data must be given, use --home to set both at once") case homeSet && dataSet: - return errors.New("-home must not be used together with -conf and -data") + return errors.New("--home must not be used together with --config and --data") case homeSet: confDir = homeDir dataDir = homeDir diff --git a/cmd/syncthing/decrypt/decrypt.go b/cmd/syncthing/decrypt/decrypt.go index 0247bb20f01..119583dbd70 100644 --- a/cmd/syncthing/decrypt/decrypt.go +++ b/cmd/syncthing/decrypt/decrypt.go @@ -45,7 +45,7 @@ func (c *CLI) Run() error { log.SetFlags(0) if c.To == "" && !c.VerifyOnly { - return fmt.Errorf("must set --to or --verify") + return fmt.Errorf("must set --to or --verify-only") } if c.TokenPath == "" { diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 00e2eef5d95..b21273ab69b 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -76,9 +76,9 @@ above). The value 0 is used to disable all of the above. The default is to show time only (2). Logging always happens to the command line (stdout) and optionally to the -file at the path specified by -logfile=path. In addition to an path, the special +file at the path specified by --logfile=path. In addition to an path, the special values "default" and "-" may be used. The former logs to DATADIR/syncthing.log -(see -data-dir), which is the default on Windows, and the latter only to stdout, +(see --data), which is the default on Windows, and the latter only to stdout, no file, which is the default anywhere else. diff --git a/lib/locations/locations.go b/lib/locations/locations.go index 3df0b877f0a..12077471ad2 100644 --- a/lib/locations/locations.go +++ b/lib/locations/locations.go @@ -40,10 +40,10 @@ const ( type BaseDirEnum string const ( - // Overridden by -home flag + // Overridden by --home flag ConfigBaseDir BaseDirEnum = "config" DataBaseDir BaseDirEnum = "data" - // User's home directory, *not* -home flag + // User's home directory, *not* --home flag UserHomeBaseDir BaseDirEnum = "userHome" LevelDBDir = "index-v0.14.0.db" @@ -98,7 +98,7 @@ var locationTemplates = map[LocationEnum]string{ HTTPSCertFile: "${config}/https-cert.pem", HTTPSKeyFile: "${config}/https-key.pem", Database: "${data}/" + LevelDBDir, - LogFile: "${data}/syncthing.log", // -logfile on Windows + LogFile: "${data}/syncthing.log", // --logfile on Windows CsrfTokens: "${data}/csrftokens.txt", PanicLog: "${data}/panic-${timestamp}.log", AuditLog: "${data}/audit-${timestamp}.log", diff --git a/lib/syncthing/utils.go b/lib/syncthing/utils.go index 3ddd6e56994..660f856d118 100644 --- a/lib/syncthing/utils.go +++ b/lib/syncthing/utils.go @@ -90,7 +90,7 @@ func LoadConfigAtStartup(path string, cert tls.Certificate, evLogger events.Logg l.Infof("Now, THAT's what we call a config from the future! Don't worry. As long as you hit that wire with the connecting hook at precisely eighty-eight miles per hour the instant the lightning strikes the tower... everything will be fine.") } if originalVersion > config.CurrentVersion && !allowNewerConfig { - return nil, fmt.Errorf("config file version (%d) is newer than supported version (%d). If this is expected, use -allow-newer-config to override.", originalVersion, config.CurrentVersion) + return nil, fmt.Errorf("config file version (%d) is newer than supported version (%d). If this is expected, use --allow-newer-config to override.", originalVersion, config.CurrentVersion) } err = archiveAndSaveConfig(cfg, originalVersion) if err != nil { diff --git a/test/cli_test.go b/test/cli_test.go index f8e783893c1..c19d4383124 100644 --- a/test/cli_test.go +++ b/test/cli_test.go @@ -31,7 +31,7 @@ func TestCLIReset(t *testing.T) { // Run reset to clean up - cmd := exec.Command("../bin/syncthing", "-no-browser", "-home", "h1", "-reset-database") + cmd := exec.Command("../bin/syncthing", "--no-browser", "--home", "h1", "--reset-database") cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout err := cmd.Run() @@ -63,9 +63,9 @@ func TestCLIGenerate(t *testing.T) { t.Fatal(err) } - // -generate should create a bunch of stuff + // --generate should create a bunch of stuff - cmd := exec.Command("../bin/syncthing", "-no-browser", "-generate", "home.out") + cmd := exec.Command("../bin/syncthing", "--no-browser", "--generate", "home.out") cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout err = cmd.Run() @@ -91,7 +91,7 @@ func TestCLIFirstStartup(t *testing.T) { // First startup should create config, BEP certificate, and HTTP certificate. - cmd := exec.Command("../bin/syncthing", "-no-browser", "-home", "home.out") + cmd := exec.Command("../bin/syncthing", "--no-browser", "--home", "home.out") cmd.Env = append(os.Environ(), "STNORESTART=1") cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout diff --git a/test/override_test.go b/test/override_test.go index a4af61bf576..b2fa14f11cd 100644 --- a/test/override_test.go +++ b/test/override_test.go @@ -199,7 +199,7 @@ func TestOverrideIgnores(t *testing.T) { log.Println("Starting sendOnly...") sendOnly := syncthingProcess{ // id1 instance: "1", - argv: []string{"-home", "h1"}, + argv: []string{"--home", "h1"}, port: 8081, apiKey: apiKey, } @@ -212,7 +212,7 @@ func TestOverrideIgnores(t *testing.T) { log.Println("Starting sendRecv...") sendRecv := syncthingProcess{ // id2 instance: "2", - argv: []string{"-home", "h2"}, + argv: []string{"--home", "h2"}, port: 8082, apiKey: apiKey, } diff --git a/test/util.go b/test/util.go index 38b00a5a704..8f69dacdaa1 100644 --- a/test/util.go +++ b/test/util.go @@ -542,7 +542,7 @@ func startInstance(t *testing.T, i int) *rc.Process { p := rc.NewProcess(addr) p.LogTo(log) - if err := p.Start("../bin/syncthing", "-home", fmt.Sprintf("h%d", i), "-no-browser"); err != nil { + if err := p.Start("../bin/syncthing", "--home", fmt.Sprintf("h%d", i), "--no-browser"); err != nil { t.Fatal(err) } p.AwaitStartup() From db15e5274346e4d3ba343c12a00bc7d1744f1b75 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Sat, 6 Nov 2021 12:38:08 +0100 Subject: [PATCH 006/220] lib/api: http.Request.BasicAuth instead of custom code (#8039) --- lib/api/api_auth.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/api/api_auth.go b/lib/api/api_auth.go index 5a009c5db00..d7d7d5a2509 100644 --- a/lib/api/api_auth.go +++ b/lib/api/api_auth.go @@ -7,9 +7,7 @@ package api import ( - "bytes" "crypto/tls" - "encoding/base64" "fmt" "net" "net/http" @@ -66,28 +64,12 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura http.Error(w, "Not Authorized", http.StatusUnauthorized) } - hdr := r.Header.Get("Authorization") - if !strings.HasPrefix(hdr, "Basic ") { + username, password, ok := r.BasicAuth() + if !ok { error() return } - hdr = hdr[6:] - bs, err := base64.StdEncoding.DecodeString(hdr) - if err != nil { - error() - return - } - - fields := bytes.SplitN(bs, []byte(":"), 2) - if len(fields) != 2 { - error() - return - } - - username := string(fields[0]) - password := string(fields[1]) - authOk := auth(username, password, guiCfg, ldapCfg) if !authOk { usernameIso := string(iso88591ToUTF8([]byte(username))) From ec8a748514e93f41e94478b59679afe67eb1d3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Colomb?= Date: Sun, 7 Nov 2021 23:59:48 +0100 Subject: [PATCH 007/220] lib/syncthing: Clean up / refactor LoadOrGenerateCertificate() utility function. (#8025) LoadOrGenerateCertificate() takes two file path arguments, but then uses the locations package to determine the actual path. Fix that with a minimally invasive change, by using the arguments instead. Factor out GenerateCertificate(). The only caller of this function is cmd/syncthing, which passes the same values, so this is technically a no-op. * lib/tlsutil: Make storing generated certificate optional. Avoid temporary cert and key files in tests, keep cert in memory. --- cmd/syncthing/main.go | 7 ++-- lib/api/api_test.go | 12 ++----- lib/connections/connections_test.go | 16 +-------- lib/discover/global_test.go | 14 ++------ lib/syncthing/syncthing_test.go | 9 +----- lib/syncthing/utils.go | 18 ++++------- lib/tlsutil/tlsutil.go | 50 +++++++++++++++++++---------- 7 files changed, 49 insertions(+), 77 deletions(-) diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index b21273ab69b..21a8fbe6bc5 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -49,16 +49,13 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/syncthing" - "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/syncthing/lib/upgrade" "github.com/pkg/errors" ) const ( - tlsDefaultCommonName = "syncthing" - deviceCertLifetimeDays = 20 * 365 - sigTerm = syscall.Signal(15) + sigTerm = syscall.Signal(15) ) const ( @@ -442,7 +439,7 @@ func generate(generateDir string, noDefaultFolder bool) error { if err == nil { l.Warnln("Key exists; will not overwrite.") } else { - cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays) + cert, err = syncthing.GenerateCertificate(certFile, keyFile) if err != nil { return errors.Wrap(err, "create certificate") } diff --git a/lib/api/api_test.go b/lib/api/api_test.go index 951776d7eac..f9ecc15c5cc 100644 --- a/lib/api/api_test.go +++ b/lib/api/api_test.go @@ -1209,15 +1209,9 @@ func TestPrefixMatch(t *testing.T) { } func TestShouldRegenerateCertificate(t *testing.T) { - dir, err := ioutil.TempDir("", "syncthing-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - // Self signed certificates expiring in less than a month are errored so we // can regenerate in time. - crt, err := tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 29) + crt, err := tlsutil.NewCertificateInMemory("foo.example.com", 29) if err != nil { t.Fatal(err) } @@ -1226,7 +1220,7 @@ func TestShouldRegenerateCertificate(t *testing.T) { } // Certificates with at least 31 days of life left are fine. - crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 31) + crt, err = tlsutil.NewCertificateInMemory("foo.example.com", 31) if err != nil { t.Fatal(err) } @@ -1236,7 +1230,7 @@ func TestShouldRegenerateCertificate(t *testing.T) { if runtime.GOOS == "darwin" { // Certificates with too long an expiry time are not allowed on macOS - crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 1000) + crt, err = tlsutil.NewCertificateInMemory("foo.example.com", 1000) if err != nil { t.Fatal(err) } diff --git a/lib/connections/connections_test.go b/lib/connections/connections_test.go index b9a10d5caa5..a7d4caf1939 100644 --- a/lib/connections/connections_test.go +++ b/lib/connections/connections_test.go @@ -11,11 +11,9 @@ import ( "crypto/tls" "errors" "fmt" - "io/ioutil" "math/rand" "net" "net/url" - "os" "strings" "testing" "time" @@ -470,21 +468,9 @@ func withConnectionPair(b *testing.B, connUri string, h func(client, server inte } func mustGetCert(b *testing.B) tls.Certificate { - f1, err := ioutil.TempFile("", "") + cert, err := tlsutil.NewCertificateInMemory("bench", 10) if err != nil { b.Fatal(err) } - f1.Close() - f2, err := ioutil.TempFile("", "") - if err != nil { - b.Fatal(err) - } - f2.Close() - cert, err := tlsutil.NewCertificate(f1.Name(), f2.Name(), "bench", 10) - if err != nil { - b.Fatal(err) - } - _ = os.Remove(f1.Name()) - _ = os.Remove(f2.Name()) return cert } diff --git a/lib/discover/global_test.go b/lib/discover/global_test.go index 8ebbc7c4d3e..8a9761782ed 100644 --- a/lib/discover/global_test.go +++ b/lib/discover/global_test.go @@ -107,13 +107,8 @@ func TestGlobalOverHTTP(t *testing.T) { } func TestGlobalOverHTTPS(t *testing.T) { - dir, err := ioutil.TempDir("", "syncthing") - if err != nil { - t.Fatal(err) - } - // Generate a server certificate. - cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 30) + cert, err := tlsutil.NewCertificateInMemory("syncthing", 30) if err != nil { t.Fatal(err) } @@ -172,13 +167,8 @@ func TestGlobalOverHTTPS(t *testing.T) { } func TestGlobalAnnounce(t *testing.T) { - dir, err := ioutil.TempDir("", "syncthing") - if err != nil { - t.Fatal(err) - } - // Generate a server certificate. - cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 30) + cert, err := tlsutil.NewCertificateInMemory("syncthing", 30) if err != nil { t.Fatal(err) } diff --git a/lib/syncthing/syncthing_test.go b/lib/syncthing/syncthing_test.go index 5c6d6278cf1..8add69ee821 100644 --- a/lib/syncthing/syncthing_test.go +++ b/lib/syncthing/syncthing_test.go @@ -9,7 +9,6 @@ package syncthing import ( "io/ioutil" "os" - "path/filepath" "testing" "time" @@ -57,13 +56,7 @@ func TestShortIDCheck(t *testing.T) { } func TestStartupFail(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "syncthing-TestStartupFail-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - cert, err := tlsutil.NewCertificate(filepath.Join(tmpDir, "cert"), filepath.Join(tmpDir, "key"), "syncthing", 365) + cert, err := tlsutil.NewCertificateInMemory("syncthing", 365) if err != nil { t.Fatal(err) } diff --git a/lib/syncthing/utils.go b/lib/syncthing/utils.go index 660f856d118..468e94a045a 100644 --- a/lib/syncthing/utils.go +++ b/lib/syncthing/utils.go @@ -25,22 +25,18 @@ import ( ) func LoadOrGenerateCertificate(certFile, keyFile string) (tls.Certificate, error) { - cert, err := tls.LoadX509KeyPair( - locations.Get(locations.CertFile), - locations.Get(locations.KeyFile), - ) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { - l.Infof("Generating ECDSA key and certificate for %s...", tlsDefaultCommonName) - return tlsutil.NewCertificate( - locations.Get(locations.CertFile), - locations.Get(locations.KeyFile), - tlsDefaultCommonName, - deviceCertLifetimeDays, - ) + return GenerateCertificate(certFile, keyFile) } return cert, nil } +func GenerateCertificate(certFile, keyFile string) (tls.Certificate, error) { + l.Infof("Generating ECDSA key and certificate for %s...", tlsDefaultCommonName) + return tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays) +} + func DefaultConfig(path string, myID protocol.DeviceID, evLogger events.Logger, noDefaultFolder bool) (config.Wrapper, error) { newCfg, err := config.NewWithFreePorts(myID) if err != nil { diff --git a/lib/tlsutil/tlsutil.go b/lib/tlsutil/tlsutil.go index 146e6d96c01..7ef22d5c73a 100644 --- a/lib/tlsutil/tlsutil.go +++ b/lib/tlsutil/tlsutil.go @@ -86,11 +86,11 @@ func SecureDefaultWithTLS12() *tls.Config { } } -// NewCertificate generates and returns a new TLS certificate. -func NewCertificate(certFile, keyFile, commonName string, lifetimeDays int) (tls.Certificate, error) { +// generateCertificate generates a PEM formatted key pair and self-signed certificate in memory. +func generateCertificate(commonName string, lifetimeDays int) (*pem.Block, *pem.Block, error) { priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) if err != nil { - return tls.Certificate{}, errors.Wrap(err, "generate key") + return nil, nil, errors.Wrap(err, "generate key") } notBefore := time.Now().Truncate(24 * time.Hour) @@ -117,19 +117,33 @@ func NewCertificate(certFile, keyFile, commonName string, lifetimeDays int) (tls derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv) if err != nil { - return tls.Certificate{}, errors.Wrap(err, "create cert") + return nil, nil, errors.Wrap(err, "create cert") + } + + certBlock := &pem.Block{Type: "CERTIFICATE", Bytes: derBytes} + keyBlock, err := pemBlockForKey(priv) + if err != nil { + return nil, nil, errors.Wrap(err, "save key") + } + + return certBlock, keyBlock, nil +} + +// NewCertificate generates and returns a new TLS certificate, saved to the given PEM files. +func NewCertificate(certFile, keyFile string, commonName string, lifetimeDays int) (tls.Certificate, error) { + certBlock, keyBlock, err := generateCertificate(commonName, lifetimeDays) + if err != nil { + return tls.Certificate{}, err } certOut, err := os.Create(certFile) if err != nil { return tls.Certificate{}, errors.Wrap(err, "save cert") } - err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - if err != nil { + if err = pem.Encode(certOut, certBlock); err != nil { return tls.Certificate{}, errors.Wrap(err, "save cert") } - err = certOut.Close() - if err != nil { + if err = certOut.Close(); err != nil { return tls.Certificate{}, errors.Wrap(err, "save cert") } @@ -137,22 +151,24 @@ func NewCertificate(certFile, keyFile, commonName string, lifetimeDays int) (tls if err != nil { return tls.Certificate{}, errors.Wrap(err, "save key") } - - block, err := pemBlockForKey(priv) - if err != nil { + if err = pem.Encode(keyOut, keyBlock); err != nil { return tls.Certificate{}, errors.Wrap(err, "save key") } - - err = pem.Encode(keyOut, block) - if err != nil { + if err = keyOut.Close(); err != nil { return tls.Certificate{}, errors.Wrap(err, "save key") } - err = keyOut.Close() + + return tls.X509KeyPair(pem.EncodeToMemory(certBlock), pem.EncodeToMemory(keyBlock)) +} + +// NewCertificateInMemory generates and returns a new TLS certificate, kept only in memory. +func NewCertificateInMemory(commonName string, lifetimeDays int) (tls.Certificate, error) { + certBlock, keyBlock, err := generateCertificate(commonName, lifetimeDays) if err != nil { - return tls.Certificate{}, errors.Wrap(err, "save key") + return tls.Certificate{}, err } - return tls.LoadX509KeyPair(certFile, keyFile) + return tls.X509KeyPair(pem.EncodeToMemory(certBlock), pem.EncodeToMemory(keyBlock)) } type DowngradingListener struct { From dec6f80d2bc9067347d80cba692da49c38cbdfde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Colomb?= Date: Mon, 8 Nov 2021 13:32:04 +0100 Subject: [PATCH 008/220] lib/config: Move the bcrypt password hashing to GUIConfiguration (#8028) What hash is used to store the password should ideally be an implementation detail, so that every user of the GUIConfiguration object automatically agrees on how to handle it. That is currently distribututed over the confighandler.go and api_auth.go files, plus tests. Add the SetHasedPassword() / CompareHashedPassword() API to keep the hashing method encapsulated. Add a separate test for it and adjust other users and tests. Remove all deprecated imports of the bcrypt package. --- lib/api/api_auth.go | 9 +++------ lib/api/api_auth_test.go | 13 +++++++------ lib/api/confighandler.go | 33 ++++++++++++++------------------- lib/config/config_test.go | 21 +++++++++++++++++++++ lib/config/guiconfiguration.go | 19 +++++++++++++++++++ 5 files changed, 64 insertions(+), 31 deletions(-) diff --git a/lib/api/api_auth.go b/lib/api/api_auth.go index d7d7d5a2509..70add9a9541 100644 --- a/lib/api/api_auth.go +++ b/lib/api/api_auth.go @@ -19,7 +19,6 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/sync" - "golang.org/x/crypto/bcrypt" ) var ( @@ -117,14 +116,12 @@ func auth(username string, password string, guiCfg config.GUIConfiguration, ldap if guiCfg.AuthMode == config.AuthModeLDAP { return authLDAP(username, password, ldapCfg) } else { - return authStatic(username, password, guiCfg.User, guiCfg.Password) + return authStatic(username, password, guiCfg) } } -func authStatic(username string, password string, configUser string, configPassword string) bool { - configPasswordBytes := []byte(configPassword) - passwordBytes := []byte(password) - return bcrypt.CompareHashAndPassword(configPasswordBytes, passwordBytes) == nil && username == configUser +func authStatic(username string, password string, guiCfg config.GUIConfiguration) bool { + return guiCfg.CompareHashedPassword(password) == nil && username == guiCfg.User } func authLDAP(username string, password string, cfg config.LDAPConfiguration) bool { diff --git a/lib/api/api_auth_test.go b/lib/api/api_auth_test.go index 4735378a5bf..046b06a0771 100644 --- a/lib/api/api_auth_test.go +++ b/lib/api/api_auth_test.go @@ -9,19 +9,20 @@ package api import ( "testing" - "golang.org/x/crypto/bcrypt" + "github.com/syncthing/syncthing/lib/config" ) -var passwordHashBytes []byte +var guiCfg config.GUIConfiguration func init() { - passwordHashBytes, _ = bcrypt.GenerateFromPassword([]byte("pass"), 0) + guiCfg.User = "user" + guiCfg.HashAndSetPassword("pass") } func TestStaticAuthOK(t *testing.T) { t.Parallel() - ok := authStatic("user", "pass", "user", string(passwordHashBytes)) + ok := authStatic("user", "pass", guiCfg) if !ok { t.Fatalf("should pass auth") } @@ -30,7 +31,7 @@ func TestStaticAuthOK(t *testing.T) { func TestSimpleAuthUsernameFail(t *testing.T) { t.Parallel() - ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes)) + ok := authStatic("userWRONG", "pass", guiCfg) if ok { t.Fatalf("should fail auth") } @@ -39,7 +40,7 @@ func TestSimpleAuthUsernameFail(t *testing.T) { func TestStaticAuthPasswordFail(t *testing.T) { t.Parallel() - ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes)) + ok := authStatic("user", "passWRONG", guiCfg) if ok { t.Fatalf("should fail auth") } diff --git a/lib/api/confighandler.go b/lib/api/confighandler.go index 8f03acfdff5..24e6b118c16 100644 --- a/lib/api/confighandler.go +++ b/lib/api/confighandler.go @@ -13,7 +13,6 @@ import ( "net/http" "github.com/julienschmidt/httprouter" - "golang.org/x/crypto/bcrypt" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/protocol" @@ -290,11 +289,13 @@ func (c *configMuxBuilder) adjustConfig(w http.ResponseWriter, r *http.Request) var errMsg string var status int waiter, err := c.cfg.Modify(func(cfg *config.Configuration) { - if to.GUI.Password, err = checkGUIPassword(cfg.GUI.Password, to.GUI.Password); err != nil { - l.Warnln("bcrypting password:", err) - errMsg = err.Error() - status = http.StatusInternalServerError - return + if to.GUI.Password != cfg.GUI.Password { + if err := to.GUI.HashAndSetPassword(to.GUI.Password); err != nil { + l.Warnln("hashing password:", err) + errMsg = err.Error() + status = http.StatusInternalServerError + return + } } *cfg = to }) @@ -370,11 +371,13 @@ func (c *configMuxBuilder) adjustGUI(w http.ResponseWriter, r *http.Request, gui var errMsg string var status int waiter, err := c.cfg.Modify(func(cfg *config.Configuration) { - if gui.Password, err = checkGUIPassword(oldPassword, gui.Password); err != nil { - l.Warnln("bcrypting password:", err) - errMsg = err.Error() - status = http.StatusInternalServerError - return + if gui.Password != oldPassword { + if err := gui.HashAndSetPassword(gui.Password); err != nil { + l.Warnln("hashing password:", err) + errMsg = err.Error() + status = http.StatusInternalServerError + return + } } cfg.GUI = gui }) @@ -418,14 +421,6 @@ func unmarshalToRawMessages(body io.ReadCloser) ([]json.RawMessage, error) { return data, err } -func checkGUIPassword(oldPassword, newPassword string) (string, error) { - if newPassword == oldPassword { - return newPassword, nil - } - hash, err := bcrypt.GenerateFromPassword([]byte(newPassword), 0) - return string(hash), err -} - func (c *configMuxBuilder) finish(w http.ResponseWriter, waiter config.Waiter) { waiter.Wait() if err := c.cfg.Save(); err != nil { diff --git a/lib/config/config_test.go b/lib/config/config_test.go index e49aa977604..5da13207c9c 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -739,6 +739,27 @@ func TestGUIConfigURL(t *testing.T) { } } +func TestGUIPasswordHash(t *testing.T) { + var c GUIConfiguration + + testPass := "pass" + if err := c.HashAndSetPassword(testPass); err != nil { + t.Fatal(err) + } + if c.Password == testPass { + t.Error("Password hashing resulted in plaintext") + } + + if err := c.CompareHashedPassword(testPass); err != nil { + t.Errorf("No match on same password: %v", err) + } + + failPass := "different" + if err := c.CompareHashedPassword(failPass); err == nil { + t.Errorf("Match on different password: %v", err) + } +} + func TestDuplicateDevices(t *testing.T) { // Duplicate devices should be removed diff --git a/lib/config/guiconfiguration.go b/lib/config/guiconfiguration.go index 7b9830b30a9..147ed9fdc36 100644 --- a/lib/config/guiconfiguration.go +++ b/lib/config/guiconfiguration.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" + "golang.org/x/crypto/bcrypt" + "github.com/syncthing/syncthing/lib/rand" ) @@ -113,6 +115,23 @@ func (c GUIConfiguration) URL() string { return u.String() } +// SetHashedPassword hashes the given plaintext password and stores the new hash. +func (c *GUIConfiguration) HashAndSetPassword(password string) error { + hash, err := bcrypt.GenerateFromPassword([]byte(password), 0) + if err != nil { + return err + } + c.Password = string(hash) + return nil +} + +// CompareHashedPassword returns nil when the given plaintext password matches the stored hash. +func (c GUIConfiguration) CompareHashedPassword(password string) error { + configPasswordBytes := []byte(c.Password) + passwordBytes := []byte(password) + return bcrypt.CompareHashAndPassword(configPasswordBytes, passwordBytes) +} + // IsValidAPIKey returns true when the given API key is valid, including both // the value in config and any overrides func (c GUIConfiguration) IsValidAPIKey(apiKey string) bool { From 591e4d8af1a60768a573dbb5409016d30d6b18b1 Mon Sep 17 00:00:00 2001 From: Simon Frei Date: Wed, 10 Nov 2021 09:46:21 +0100 Subject: [PATCH 009/220] gui, lib: Fix tracking deleted locally-changed on encrypted (fixes #7715) (#7726) --- gui/default/index.html | 6 - .../syncthing/core/syncthingController.js | 23 +--- lib/db/lowlevel.go | 117 ++++++++++++++---- lib/db/set.go | 16 +++ lib/model/fakeconns_test.go | 1 + lib/model/folder.go | 84 ++++++++----- lib/model/folder_recvonly_test.go | 37 ++---- lib/model/folder_sendrecv_test.go | 16 ++- lib/model/model.go | 4 +- lib/model/model_test.go | 74 ++++++++--- lib/model/requests_test.go | 20 +-- lib/model/testutils_test.go | 15 +++ 12 files changed, 271 insertions(+), 142 deletions(-) diff --git a/gui/default/index.html b/gui/default/index.html index 07cd90d6946..43ae81434db 100644 --- a/gui/default/index.html +++ b/gui/default/index.html @@ -455,12 +455,6 @@

{{model[folder.id].receiveOnlyTotalItems | alwaysNumber | localeNumber}} items, ~{{model[folder.id].receiveOnlyChangedBytes | binary}}B - -  Locally Changed Items - - {{receiveEncryptedItemsCount(folder) | alwaysNumber | localeNumber}} items, ~{{model[folder.id].receiveOnlyChangedBytes | binary}}B - -  Folder Type diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index 8e91fda4190..fa565b8b8d3 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -931,9 +931,9 @@ angular.module('syncthing.core') return 'faileditems'; } if ($scope.hasReceiveOnlyChanged(folderCfg)) { - return 'localadditions'; - } - if ($scope.hasReceiveEncryptedItems(folderCfg)) { + if (folderCfg.type === "receiveonly") { + return 'localadditions'; + } return 'localunencrypted'; } if (folderCfg.devices.length <= 1) { @@ -2609,28 +2609,13 @@ angular.module('syncthing.core') }; $scope.hasReceiveOnlyChanged = function (folderCfg) { - if (!folderCfg || folderCfg.type !== "receiveonly") { + if (!folderCfg || folderCfg.type !== ["receiveonly", "receiveencrypted"].indexOf(folderCfg.type) === -1) { return false; } var counts = $scope.model[folderCfg.id]; return counts && counts.receiveOnlyTotalItems > 0; }; - $scope.hasReceiveEncryptedItems = function (folderCfg) { - if (!folderCfg || folderCfg.type !== "receiveencrypted") { - return false; - } - return $scope.receiveEncryptedItemsCount(folderCfg) > 0; - }; - - $scope.receiveEncryptedItemsCount = function (folderCfg) { - var counts = $scope.model[folderCfg.id]; - if (!counts) { - return 0; - } - return counts.receiveOnlyTotalItems - counts.receiveOnlyChangedDeletes; - } - $scope.revertOverride = function () { $http.post( urlbase + "/db/" + $scope.revertOverrideParams.operation +"?folder=" diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index 7f4916da488..88792a36d56 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -223,35 +223,10 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta blocksHashSame := ok && bytes.Equal(ef.BlocksHash, f.BlocksHash) if ok { - if len(ef.Blocks) != 0 && !ef.IsInvalid() && ef.Size > 0 { - for _, block := range ef.Blocks { - keyBuf, err = db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name) - if err != nil { - return err - } - if err := t.Delete(keyBuf); err != nil { - return err - } - } - if !blocksHashSame { - keyBuf, err := db.keyer.GenerateBlockListMapKey(keyBuf, folder, ef.BlocksHash, name) - if err != nil { - return err - } - if err = t.Delete(keyBuf); err != nil { - return err - } - } - } - - keyBuf, err = db.keyer.GenerateSequenceKey(keyBuf, folder, ef.SequenceNo()) + keyBuf, err = db.removeLocalBlockAndSequenceInfo(keyBuf, folder, name, ef, !blocksHashSame, &t) if err != nil { return err } - if err := t.Delete(keyBuf); err != nil { - return err - } - l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, ef.SequenceNo(), ef.FileName()) } f.Sequence = meta.nextLocalSeq() @@ -314,6 +289,96 @@ func (db *Lowlevel) updateLocalFiles(folder []byte, fs []protocol.FileInfo, meta return t.Commit() } +func (db *Lowlevel) removeLocalFiles(folder []byte, nameStrs []string, meta *metadataTracker) error { + db.gcMut.RLock() + defer db.gcMut.RUnlock() + + t, err := db.newReadWriteTransaction(meta.CommitHook(folder)) + if err != nil { + return err + } + defer t.close() + + var dk, gk, buf []byte + for _, nameStr := range nameStrs { + name := []byte(nameStr) + dk, err = db.keyer.GenerateDeviceFileKey(dk, folder, protocol.LocalDeviceID[:], name) + if err != nil { + return err + } + + ef, ok, err := t.getFileByKey(dk) + if err != nil { + return err + } + if !ok { + l.Debugf("remove (local); folder=%q %v: file doesn't exist", folder, nameStr) + continue + } + + buf, err = db.removeLocalBlockAndSequenceInfo(buf, folder, name, ef, true, &t) + if err != nil { + return err + } + + meta.removeFile(protocol.LocalDeviceID, ef) + + gk, err = db.keyer.GenerateGlobalVersionKey(gk, folder, name) + if err != nil { + return err + } + buf, err = t.removeFromGlobal(gk, buf, folder, protocol.LocalDeviceID[:], name, meta) + if err != nil { + return err + } + + err = t.Delete(dk) + if err != nil { + return err + } + + if err := t.Checkpoint(); err != nil { + return err + } + } + + return t.Commit() +} + +func (db *Lowlevel) removeLocalBlockAndSequenceInfo(keyBuf, folder, name []byte, ef protocol.FileInfo, removeFromBlockListMap bool, t *readWriteTransaction) ([]byte, error) { + var err error + if len(ef.Blocks) != 0 && !ef.IsInvalid() && ef.Size > 0 { + for _, block := range ef.Blocks { + keyBuf, err = db.keyer.GenerateBlockMapKey(keyBuf, folder, block.Hash, name) + if err != nil { + return nil, err + } + if err := t.Delete(keyBuf); err != nil { + return nil, err + } + } + if removeFromBlockListMap { + keyBuf, err := db.keyer.GenerateBlockListMapKey(keyBuf, folder, ef.BlocksHash, name) + if err != nil { + return nil, err + } + if err = t.Delete(keyBuf); err != nil { + return nil, err + } + } + } + + keyBuf, err = db.keyer.GenerateSequenceKey(keyBuf, folder, ef.SequenceNo()) + if err != nil { + return nil, err + } + if err := t.Delete(keyBuf); err != nil { + return nil, err + } + l.Debugf("removing sequence; folder=%q sequence=%v %v", folder, ef.SequenceNo(), ef.FileName()) + return keyBuf, nil +} + func (db *Lowlevel) dropFolder(folder []byte) error { db.gcMut.RLock() defer db.gcMut.RUnlock() diff --git a/lib/db/set.go b/lib/db/set.go index a1c0cb64962..12a7083cb8f 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -144,6 +144,22 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) { } } +func (s *FileSet) RemoveLocalItems(items []string) { + opStr := fmt.Sprintf("%s RemoveLocalItems([%d])", s.folder, len(items)) + l.Debugf(opStr) + + s.updateMutex.Lock() + defer s.updateMutex.Unlock() + + for i := range items { + items[i] = osutil.NormalizedFilename(items[i]) + } + + if err := s.db.removeLocalFiles([]byte(s.folder), items, s.meta); err != nil && !backend.IsClosed(err) { + fatalError(err, opStr, s.db) + } +} + type Snapshot struct { folder string t readOnlyTransaction diff --git a/lib/model/fakeconns_test.go b/lib/model/fakeconns_test.go index 79b4e85520e..9b5495b5f17 100644 --- a/lib/model/fakeconns_test.go +++ b/lib/model/fakeconns_test.go @@ -162,6 +162,7 @@ func (f *fakeConnection) sendIndexUpdate() { func addFakeConn(m *testModel, dev protocol.DeviceID, folderID string) *fakeConnection { fc := newFakeConnection(dev, m) + fc.folder = folderID m.AddConnection(fc, protocol.Hello{}) m.ClusterConfig(dev, protocol.ClusterConfig{ diff --git a/lib/model/folder.go b/lib/model/folder.go index 18ef41a0657..41b9fe98067 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -532,16 +532,20 @@ func (f *folder) scanSubdirs(subDirs []string) error { return nil } +const maxToRemove = 1000 + type scanBatch struct { - *db.FileInfoBatch - f *folder + f *folder + updateBatch *db.FileInfoBatch + toRemove []string } func (f *folder) newScanBatch() *scanBatch { b := &scanBatch{ - f: f, + f: f, + toRemove: make([]string, 0, maxToRemove), } - b.FileInfoBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { + b.updateBatch = db.NewFileInfoBatch(func(fs []protocol.FileInfo) error { if err := b.f.getHealthErrorWithoutIgnores(); err != nil { l.Debugf("Stopping scan of folder %s due to: %s", b.f.Description(), err) return err @@ -552,9 +556,32 @@ func (f *folder) newScanBatch() *scanBatch { return b } -// Append adds the fileinfo to the batch for updating, and does a few checks. +func (b *scanBatch) Remove(item string) { + b.toRemove = append(b.toRemove, item) +} + +func (b *scanBatch) flushToRemove() { + if len(b.toRemove) > 0 { + b.f.fset.RemoveLocalItems(b.toRemove) + b.toRemove = b.toRemove[:0] + } +} + +func (b *scanBatch) Flush() error { + b.flushToRemove() + return b.updateBatch.Flush() +} + +func (b *scanBatch) FlushIfFull() error { + if len(b.toRemove) >= maxToRemove { + b.flushToRemove() + } + return b.updateBatch.FlushIfFull() +} + +// Update adds the fileinfo to the batch for updating, and does a few checks. // It returns false if the checks result in the file not going to be updated or removed. -func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { +func (b *scanBatch) Update(fi protocol.FileInfo, snap *db.Snapshot) bool { // Check for a "virtual" parent directory of encrypted files. We don't track // it, but check if anything still exists within and delete it otherwise. if b.f.Type == config.FolderTypeReceiveEncrypted && fi.IsDirectory() && protocol.IsEncryptedParent(fs.PathComponents(fi.Name)) { @@ -565,20 +592,21 @@ func (b *scanBatch) Append(fi protocol.FileInfo, snap *db.Snapshot) bool { } // Resolve receive-only items which are identical with the global state or // the global item is our own receive-only item. - // Except if they are in a receive-encrypted folder and are locally added. - // Those must never be sent in index updates and thus must retain the flag. switch gf, ok := snap.GetGlobal(fi.Name); { case !ok: case gf.IsReceiveOnlyChanged(): - if b.f.Type == config.FolderTypeReceiveOnly && fi.Deleted { - l.Debugf("%v scanning: Marking deleted item as not locally changed", b.f, fi) - fi.LocalFlags &^= protocol.FlagLocalReceiveOnly + if fi.IsDeleted() { + // Our item is deleted and the global item is our own receive only + // file. No point in keeping track of that. + b.Remove(fi.Name) + return true } case gf.IsEquivalentOptional(fi, b.f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly): - l.Debugf("%v scanning: Replacing scanned file info with global as it's equivalent", b.f, fi) + // What we have locally is equivalent to the global file. + l.Debugf("%v scanning: Merging identical locally changed item with global", b.f, fi) fi = gf } - b.FileInfoBatch.Append(fi) + b.updateBatch.Append(fi) return true } @@ -634,7 +662,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i return changes, err } - if batch.Append(res.File, snap) { + if batch.Update(res.File, snap) { changes++ } @@ -642,7 +670,7 @@ func (f *folder) scanSubdirsChangedAndNew(subDirs []string, batch *scanBatch) (i case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: default: if nf, ok := f.findRename(snap, res.File, alreadyUsedOrExisting); ok { - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } } @@ -683,7 +711,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch for _, file := range toIgnore { l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } if err := batch.FlushIfFull(); err != nil { @@ -713,7 +741,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch l.Debugln("marking file as ignored", file) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } @@ -743,24 +771,24 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch nf.Version = protocol.Vector{} } l.Debugln("marking file as deleted", nf) - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } case file.IsDeleted() && file.IsReceiveOnlyChanged(): switch f.Type { case config.FolderTypeReceiveOnly, config.FolderTypeReceiveEncrypted: - if gf, _ := snap.GetGlobal(file.Name); gf.IsDeleted() { + switch gf, ok := snap.GetGlobal(file.Name); { + case !ok: + case gf.IsReceiveOnlyChanged(): + l.Debugln("removing deleted, receive-only item that is globally receive-only from db", file) + batch.Remove(file.Name) + changes++ + case gf.IsDeleted(): // Our item is deleted and the global item is deleted too. We just // pretend it is a normal deleted file (nobody cares about that). - // Except if this is a receive-encrypted folder and it - // is a locally added file. Those must never be sent - // in index updates and thus must retain the flag. - if f.Type == config.FolderTypeReceiveEncrypted && gf.IsReceiveOnlyChanged() { - return true - } l.Debugf("%v scanning: Marking globally deleted item as not locally changed: %v", f, file.Name) file.LocalFlags &^= protocol.FlagLocalReceiveOnly - if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + if batch.Update(file.ConvertDeletedToFileInfo(), snap) { changes++ } } @@ -769,7 +797,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch // deleted and just the folder type/local flags changed. file.LocalFlags &^= protocol.FlagLocalReceiveOnly l.Debugln("removing receive-only flag on deleted item", file) - if batch.Append(file.ConvertDeletedToFileInfo(), snap) { + if batch.Update(file.ConvertDeletedToFileInfo(), snap) { changes++ } } @@ -788,7 +816,7 @@ func (f *folder) scanSubdirsDeletedAndIgnored(subDirs []string, batch *scanBatch for _, file := range toIgnore { l.Debugln("marking file as ignored", f) nf := file.ConvertToIgnoredFileInfo() - if batch.Append(nf, snap) { + if batch.Update(nf, snap) { changes++ } if iterError = batch.FlushIfFull(); iterError != nil { diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go index 9a48c78f23c..740a31202c1 100644 --- a/lib/model/folder_recvonly_test.go +++ b/lib/model/folder_recvonly_test.go @@ -37,9 +37,9 @@ func TestRecvOnlyRevertDeletes(t *testing.T) { for _, dir := range []string{".stfolder", "ignDir", "unknownDir"} { must(t, ffs.MkdirAll(dir, 0755)) } - must(t, writeFile(ffs, "ignDir/ignFile", []byte("hello\n"), 0644)) - must(t, writeFile(ffs, "unknownDir/unknownFile", []byte("hello\n"), 0644)) - must(t, writeFile(ffs, ".stignore", []byte("ignDir\n"), 0644)) + writeFilePerm(t, ffs, "ignDir/ignFile", []byte("hello\n"), 0644) + writeFilePerm(t, ffs, "unknownDir/unknownFile", []byte("hello\n"), 0644) + writeFilePerm(t, ffs, ".stignore", []byte("ignDir\n"), 0644) knownFiles := setupKnownFiles(t, ffs, []byte("hello\n")) @@ -151,7 +151,7 @@ func TestRecvOnlyRevertNeeds(t *testing.T) { // Update the file. newData := []byte("totally different data\n") - must(t, writeFile(ffs, "knownDir/knownFile", newData, 0644)) + writeFilePerm(t, ffs, "knownDir/knownFile", newData, 0644) // Rescan. @@ -241,8 +241,8 @@ func TestRecvOnlyUndoChanges(t *testing.T) { // Create a file and modify another const file = "foo" - must(t, writeFile(ffs, file, []byte("hello\n"), 0644)) - must(t, writeFile(ffs, "knownDir/knownFile", []byte("bye\n"), 0644)) + writeFilePerm(t, ffs, file, []byte("hello\n"), 0644) + writeFilePerm(t, ffs, "knownDir/knownFile", []byte("bye\n"), 0644) must(t, m.ScanFolder("ro")) @@ -254,7 +254,7 @@ func TestRecvOnlyUndoChanges(t *testing.T) { // Remove the file again and undo the modification must(t, ffs.Remove(file)) - must(t, writeFile(ffs, "knownDir/knownFile", oldData, 0644)) + writeFilePerm(t, ffs, "knownDir/knownFile", oldData, 0644) must(t, ffs.Chtimes("knownDir/knownFile", knownFiles[1].ModTime(), knownFiles[1].ModTime())) must(t, m.ScanFolder("ro")) @@ -377,8 +377,8 @@ func TestRecvOnlyRemoteUndoChanges(t *testing.T) { const file = "foo" knownFile := filepath.Join("knownDir", "knownFile") - must(t, writeFile(ffs, file, []byte("hello\n"), 0644)) - must(t, writeFile(ffs, knownFile, []byte("bye\n"), 0644)) + writeFilePerm(t, ffs, file, []byte("hello\n"), 0644) + writeFilePerm(t, ffs, knownFile, []byte("bye\n"), 0644) must(t, m.ScanFolder("ro")) @@ -434,7 +434,7 @@ func TestRecvOnlyRevertOwnID(t *testing.T) { must(t, ffs.MkdirAll(".stfolder", 0755)) data := []byte("hello\n") name := "foo" - must(t, writeFile(ffs, name, data, 0644)) + writeFilePerm(t, ffs, name, data, 0644) // Make sure the file is scanned and locally changed must(t, m.ScanFolder("ro")) @@ -484,7 +484,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi t.Helper() must(t, ffs.MkdirAll("knownDir", 0755)) - must(t, writeFile(ffs, "knownDir/knownFile", data, 0644)) + writeFilePerm(t, ffs, "knownDir/knownFile", data, 0644) t0 := time.Now().Add(-1 * time.Minute) must(t, ffs.Chtimes("knownDir/knownFile", t0, t0)) @@ -541,18 +541,3 @@ func setupROFolder(t *testing.T) (*testModel, *receiveOnlyFolder, context.Cancel return m, f, cancel } - -func writeFile(fs fs.Filesystem, filename string, data []byte, perm fs.FileMode) error { - fd, err := fs.Create(filename) - if err != nil { - return err - } - _, err = fd.Write(data) - if err != nil { - return err - } - if err := fd.Close(); err != nil { - return err - } - return fs.Chmod(filename, perm) -} diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 2894fe14f59..06bf7d6d79d 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -77,12 +77,10 @@ func setupFile(filename string, blockNumbers []int) protocol.FileInfo { } } -func createFile(t *testing.T, name string, fs fs.Filesystem) protocol.FileInfo { +func createEmptyFileInfo(t *testing.T, name string, fs fs.Filesystem) protocol.FileInfo { t.Helper() - f, err := fs.Create(name) - must(t, err) - f.Close() + writeFile(t, fs, name, nil) fi, err := fs.Stat(name) must(t, err) file, err := scanner.CreateFileInfo(fi, name, fs) @@ -913,7 +911,7 @@ func TestSRConflictReplaceFileByDir(t *testing.T) { name := "foo" // create local file - file := createFile(t, name, ffs) + file := createEmptyFileInfo(t, name, ffs) file.Version = protocol.Vector{}.Update(myID.Short()) f.updateLocalsFromScanning([]protocol.FileInfo{file}) @@ -945,7 +943,7 @@ func TestSRConflictReplaceFileByLink(t *testing.T) { name := "foo" // create local file - file := createFile(t, name, ffs) + file := createEmptyFileInfo(t, name, ffs) file.Version = protocol.Vector{}.Update(myID.Short()) f.updateLocalsFromScanning([]protocol.FileInfo{file}) @@ -983,7 +981,7 @@ func TestDeleteBehindSymlink(t *testing.T) { file := filepath.Join(link, "file") must(t, ffs.MkdirAll(link, 0755)) - fi := createFile(t, file, ffs) + fi := createEmptyFileInfo(t, file, ffs) f.updateLocalsFromScanning([]protocol.FileInfo{fi}) must(t, osutil.RenameOrCopy(fs.CopyRangeMethodStandard, ffs, destFs, file, "file")) must(t, ffs.RemoveAll(link)) @@ -1099,7 +1097,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) { name := "foo" contents := []byte("contents") - must(t, writeFile(ffs, name, contents, 0644)) + writeFile(t, ffs, name, contents) must(t, f.scanSubdirs(nil)) var cur protocol.FileInfo @@ -1122,7 +1120,7 @@ func TestPullCaseOnlyPerformFinish(t *testing.T) { remote.Version = protocol.Vector{}.Update(device1.Short()) remote.Name = strings.ToUpper(cur.Name) temp := fs.TempName(remote.Name) - must(t, writeFile(ffs, temp, contents, 0644)) + writeFile(t, ffs, temp, contents) scanChan := make(chan string, 1) dbUpdateChan := make(chan dbUpdateJob, 1) diff --git a/lib/model/model.go b/lib/model/model.go index c1b3ddaa810..28a3a5469e8 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -1053,7 +1053,6 @@ func (m *model) RemoteNeedFolderFiles(folder string, device protocol.DeviceID, p func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, error) { m.fmut.RLock() rf, ok := m.folderFiles[folder] - cfg := m.folderCfgs[folder] m.fmut.RUnlock() if !ok { @@ -1071,11 +1070,10 @@ func (m *model) LocalChangedFolderFiles(folder string, page, perpage int) ([]db. } p := newPager(page, perpage) - recvEnc := cfg.Type == config.FolderTypeReceiveEncrypted files := make([]db.FileInfoTruncated, 0, perpage) snap.WithHaveTruncated(protocol.LocalDeviceID, func(f protocol.FileIntf) bool { - if !f.IsReceiveOnlyChanged() || (recvEnc && f.IsDeleted()) { + if !f.IsReceiveOnlyChanged() { return true } if p.skip() { diff --git a/lib/model/model_test.go b/lib/model/model_test.go index b665abc01d7..121bcdffeec 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -299,7 +299,7 @@ func BenchmarkRequestInSingleFile(b *testing.B) { mustRemove(b, defaultFs.RemoveAll("request")) defer func() { mustRemove(b, defaultFs.RemoveAll("request")) }() must(b, defaultFs.MkdirAll("request/for/a/file/in/a/couple/of/dirs", 0755)) - writeFile(defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf, 0644) + writeFile(b, defaultFs, "request/for/a/file/in/a/couple/of/dirs/128k", buf) b.ResetTimer() @@ -1489,7 +1489,7 @@ func TestIgnores(t *testing.T) { // Assure a clean start state mustRemove(t, defaultFs.RemoveAll(config.DefaultMarkerName)) mustRemove(t, defaultFs.MkdirAll(config.DefaultMarkerName, 0644)) - writeFile(defaultFs, ".stignore", []byte(".*\nquux\n"), 0644) + writeFile(t, defaultFs, ".stignore", []byte(".*\nquux\n")) m := setupModel(t, defaultCfgWrapper) defer cleanupModel(m) @@ -2030,8 +2030,8 @@ func benchmarkTree(b *testing.B, n1, n2 int) { func TestIssue3028(t *testing.T) { // Create two files that we'll delete, one with a name that is a prefix of the other. - must(t, writeFile(defaultFs, "testrm", []byte("Hello"), 0644)) - must(t, writeFile(defaultFs, "testrm2", []byte("Hello"), 0644)) + writeFile(t, defaultFs, "testrm", []byte("Hello")) + writeFile(t, defaultFs, "testrm2", []byte("Hello")) defer func() { mustRemove(t, defaultFs.Remove("testrm")) mustRemove(t, defaultFs.Remove("testrm2")) @@ -3403,7 +3403,7 @@ func TestRenameSequenceOrder(t *testing.T) { ffs := fcfg.Filesystem() for i := 0; i < numFiles; i++ { v := fmt.Sprintf("%d", i) - must(t, writeFile(ffs, v, []byte(v), 0644)) + writeFile(t, ffs, v, []byte(v)) } m.ScanFolders() @@ -3426,7 +3426,7 @@ func TestRenameSequenceOrder(t *testing.T) { continue } v := fmt.Sprintf("%d", i) - must(t, writeFile(ffs, v, []byte(v+"-new"), 0644)) + writeFile(t, ffs, v, []byte(v+"-new")) } // Rename must(t, ffs.Rename("3", "17")) @@ -3470,7 +3470,7 @@ func TestRenameSameFile(t *testing.T) { defer cleanupModel(m) ffs := fcfg.Filesystem() - must(t, writeFile(ffs, "file", []byte("file"), 0644)) + writeFile(t, ffs, "file", []byte("file")) m.ScanFolders() @@ -3522,8 +3522,8 @@ func TestRenameEmptyFile(t *testing.T) { ffs := fcfg.Filesystem() - must(t, writeFile(ffs, "file", []byte("data"), 0644)) - must(t, writeFile(ffs, "empty", nil, 0644)) + writeFile(t, ffs, "file", []byte("data")) + writeFile(t, ffs, "empty", nil) m.ScanFolders() @@ -3598,11 +3598,11 @@ func TestBlockListMap(t *testing.T) { defer cleanupModel(m) ffs := fcfg.Filesystem() - must(t, writeFile(ffs, "one", []byte("content"), 0644)) - must(t, writeFile(ffs, "two", []byte("content"), 0644)) - must(t, writeFile(ffs, "three", []byte("content"), 0644)) - must(t, writeFile(ffs, "four", []byte("content"), 0644)) - must(t, writeFile(ffs, "five", []byte("content"), 0644)) + writeFile(t, ffs, "one", []byte("content")) + writeFile(t, ffs, "two", []byte("content")) + writeFile(t, ffs, "three", []byte("content")) + writeFile(t, ffs, "four", []byte("content")) + writeFile(t, ffs, "five", []byte("content")) m.ScanFolders() @@ -3631,7 +3631,7 @@ func TestBlockListMap(t *testing.T) { // Modify must(t, ffs.Remove("two")) - must(t, writeFile(ffs, "two", []byte("mew-content"), 0644)) + writeFile(t, ffs, "two", []byte("mew-content")) // Rename must(t, ffs.Rename("three", "new-three")) @@ -3667,7 +3667,7 @@ func TestScanRenameCaseOnly(t *testing.T) { ffs := fcfg.Filesystem() name := "foo" - must(t, writeFile(ffs, name, []byte("contents"), 0644)) + writeFile(t, ffs, name, []byte("contents")) m.ScanFolders() @@ -3791,7 +3791,7 @@ func TestScanDeletedROChangedOnSR(t *testing.T) { name := "foo" - must(t, writeFile(ffs, name, []byte(name), 0644)) + writeFile(t, ffs, name, []byte(name)) m.ScanFolders() file, ok := m.testCurrentFolderFile(fcfg.ID, name) @@ -4255,6 +4255,46 @@ func TestPendingFolder(t *testing.T) { } } +func TestDeletedNotLocallyChangedReceiveOnly(t *testing.T) { + deletedNotLocallyChanged(t, config.FolderTypeReceiveOnly) +} + +func TestDeletedNotLocallyChangedReceiveEncrypted(t *testing.T) { + deletedNotLocallyChanged(t, config.FolderTypeReceiveEncrypted) +} + +func deletedNotLocallyChanged(t *testing.T, ft config.FolderType) { + w, fcfg, wCancel := tmpDefaultWrapper() + tfs := fcfg.Filesystem() + fcfg.Type = ft + setFolder(t, w, fcfg) + defer wCancel() + m := setupModel(t, w) + defer cleanupModelAndRemoveDir(m, tfs.URI()) + + name := "foo" + writeFile(t, tfs, name, nil) + must(t, m.ScanFolder(fcfg.ID)) + + fi, ok, err := m.CurrentFolderFile(fcfg.ID, name) + must(t, err) + if !ok { + t.Fatal("File hasn't been added") + } + if !fi.IsReceiveOnlyChanged() { + t.Fatal("File isn't receive-only-changed") + } + + must(t, tfs.Remove(name)) + must(t, m.ScanFolder(fcfg.ID)) + + _, ok, err = m.CurrentFolderFile(fcfg.ID, name) + must(t, err) + if ok { + t.Error("Expected file to be removed from db") + } +} + func equalStringsInAnyOrder(a, b []string) bool { if len(a) != len(b) { return false diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index a0c491d71b6..35df4fdd9e9 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -330,9 +330,7 @@ func pullInvalidIgnored(t *testing.T, ft config.FolderType) { fc.deleteFile(invDel) fc.addFile(ign, 0644, protocol.FileInfoTypeFile, contents) fc.addFile(ignExisting, 0644, protocol.FileInfoTypeFile, contents) - if err := writeFile(fss, ignExisting, otherContents, 0644); err != nil { - panic(err) - } + writeFile(t, fss, ignExisting, otherContents) done := make(chan struct{}) fc.setIndexFn(func(_ context.Context, folder string, fs []protocol.FileInfo) error { @@ -486,7 +484,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) { payload := []byte("hello") - must(t, writeFile(tfs, "foo", payload, 0777)) + writeFile(t, tfs, "foo", payload) received := make(chan []protocol.FileInfo) fc.setIndexFn(func(_ context.Context, _ string, fs []protocol.FileInfo) error { @@ -526,7 +524,7 @@ func TestRescanIfHaveInvalidContent(t *testing.T) { payload = []byte("bye") buf = make([]byte, len(payload)) - must(t, writeFile(tfs, "foo", payload, 0777)) + writeFile(t, tfs, "foo", payload) _, err = m.Request(device1, "default", "foo", 0, int32(len(payload)), 0, f.Blocks[0].Hash, f.Blocks[0].WeakHash, false) if err == nil { @@ -1066,9 +1064,7 @@ func TestIgnoreDeleteUnignore(t *testing.T) { return nil }) - if err := writeFile(fss, file, contents, 0644); err != nil { - panic(err) - } + writeFile(t, fss, file, contents) m.ScanFolders() select { @@ -1430,6 +1426,14 @@ func TestRequestReceiveEncrypted(t *testing.T) { // Simulate request from device that is untrusted too, i.e. with non-empty, but garbage hash _, err := m.Request(device1, fcfg.ID, name, 0, 1064, 0, []byte("garbage"), 0, false) must(t, err) + + changed, err := m.LocalChangedFolderFiles(fcfg.ID, 1, 10) + must(t, err) + if l := len(changed); l != 1 { + t.Errorf("Expected one locally changed file, got %v", l) + } else if changed[0].Name != files[0].Name { + t.Errorf("Expected %v, got %v", files[0].Name, changed[0].Name) + } } func TestRequestGlobalInvalidToValid(t *testing.T) { diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index 6c2a29071e6..4ee390f5d8e 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -435,3 +435,18 @@ func addDevice2(t testing.TB, w config.Wrapper, fcfg config.FolderConfiguration) must(t, err) waiter.Wait() } + +func writeFile(t testing.TB, filesystem fs.Filesystem, name string, data []byte) { + t.Helper() + fd, err := filesystem.Create(name) + must(t, err) + defer fd.Close() + _, err = fd.Write(data) + must(t, err) +} + +func writeFilePerm(t testing.TB, filesystem fs.Filesystem, name string, data []byte, perm fs.FileMode) { + t.Helper() + writeFile(t, filesystem, name, data) + must(t, filesystem.Chmod(name, perm)) +} From f1bf4d899a0853aefc1ec869df07ecbc54b49f01 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 17 Nov 2021 11:42:20 +0100 Subject: [PATCH 010/220] lib/model: Correct handling of fakefs cache We looked under one cache key, then stored under another... --- lib/fs/fakefs.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/fs/fakefs.go b/lib/fs/fakefs.go index 4d75ab0f100..7bc356d98da 100644 --- a/lib/fs/fakefs.go +++ b/lib/fs/fakefs.go @@ -89,11 +89,9 @@ func newFakeFilesystem(rootURI string, _ ...Option) *fakeFS { fakeFSMut.Lock() defer fakeFSMut.Unlock() - root := rootURI var params url.Values uri, err := url.Parse(rootURI) if err == nil { - root = uri.Path params = uri.Query() } @@ -157,7 +155,7 @@ func newFakeFilesystem(rootURI string, _ ...Option) *fakeFS { // the filesystem initially. fs.latency, _ = time.ParseDuration(params.Get("latency")) - fakeFSCache[root] = fs + fakeFSCache[rootURI] = fs return fs } From 12fb7f2a0a58d720718df8d67bb959090d1e9ec2 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Wed, 17 Nov 2021 12:10:06 +0100 Subject: [PATCH 011/220] lib/model: Correct "reverting folder" log entry --- lib/model/folder_recvonly.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go index ee6f0c99a1f..827fbe5d863 100644 --- a/lib/model/folder_recvonly.go +++ b/lib/model/folder_recvonly.go @@ -68,7 +68,7 @@ func (f *receiveOnlyFolder) Revert() { } func (f *receiveOnlyFolder) revert() error { - l.Infof("Reverting folder %v", f.Description) + l.Infof("Reverting folder %v", f.Description()) f.setState(FolderScanning) defer f.setState(FolderIdle) From 100870e14232985962f2fcb73ad531f0e6eb9512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Colomb?= Date: Thu, 18 Nov 2021 22:57:59 +0100 Subject: [PATCH 012/220] cmd/syncthing: Implement generate as a subcommand with optional API credential setting (fixes #8021) (#8043) Accept a subcommand as an alternative to the --generate option. It accepts a custom config directory through either the --home or --config options, using the default location if neither is given. Add the options --gui-user and --gui-password to "generate", but not the "serve --generate" option form. If either is given, an existing config will not abort the command, but rather load, modify and save it with the new credentials. The password can be read from standard input by passing only a single dash as argument. Config modification is skipped if the value matches what's already in the config. * cmd/syncthing: Utilize lib/locations package in generate(). Instead of manually joining paths with "magic" file names, get them from the centralized locations helper lib. * cmd/syncthing: Simplify logging for --generate option. Visible change: No more timestamp prefixes. --- cmd/syncthing/cmdutil/options_common.go | 15 ++ cmd/syncthing/{ => cmdutil}/options_others.go | 4 +- .../{ => cmdutil}/options_windows.go | 4 +- cmd/syncthing/generate/generate.go | 144 ++++++++++++++++++ cmd/syncthing/main.go | 86 ++--------- lib/syncthing/utils.go | 23 +++ 6 files changed, 197 insertions(+), 79 deletions(-) create mode 100644 cmd/syncthing/cmdutil/options_common.go rename cmd/syncthing/{ => cmdutil}/options_others.go (86%) rename cmd/syncthing/{ => cmdutil}/options_windows.go (86%) create mode 100644 cmd/syncthing/generate/generate.go diff --git a/cmd/syncthing/cmdutil/options_common.go b/cmd/syncthing/cmdutil/options_common.go new file mode 100644 index 00000000000..34485e6a563 --- /dev/null +++ b/cmd/syncthing/cmdutil/options_common.go @@ -0,0 +1,15 @@ +// Copyright (C) 2021 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package cmdutil + +// CommonOptions are reused among several subcommands +type CommonOptions struct { + buildCommonOptions + ConfDir string `name:"config" placeholder:"PATH" help:"Set configuration directory (config and keys)"` + HomeDir string `name:"home" placeholder:"PATH" help:"Set configuration and data directory"` + NoDefaultFolder bool `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"` +} diff --git a/cmd/syncthing/options_others.go b/cmd/syncthing/cmdutil/options_others.go similarity index 86% rename from cmd/syncthing/options_others.go rename to cmd/syncthing/cmdutil/options_others.go index b1f4764cadf..e299ba48cf9 100644 --- a/cmd/syncthing/options_others.go +++ b/cmd/syncthing/cmdutil/options_others.go @@ -7,8 +7,8 @@ //go:build !windows // +build !windows -package main +package cmdutil -type buildServeOptions struct { +type buildCommonOptions struct { HideConsole bool `hidden:""` } diff --git a/cmd/syncthing/options_windows.go b/cmd/syncthing/cmdutil/options_windows.go similarity index 86% rename from cmd/syncthing/options_windows.go rename to cmd/syncthing/cmdutil/options_windows.go index 38d26a22a1e..99fb50d5b96 100644 --- a/cmd/syncthing/options_windows.go +++ b/cmd/syncthing/cmdutil/options_windows.go @@ -4,8 +4,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package main +package cmdutil -type buildServeOptions struct { +type buildCommonOptions struct { HideConsole bool `name:"no-console" help:"Hide console window"` } diff --git a/cmd/syncthing/generate/generate.go b/cmd/syncthing/generate/generate.go new file mode 100644 index 00000000000..8c5bd1323fe --- /dev/null +++ b/cmd/syncthing/generate/generate.go @@ -0,0 +1,144 @@ +// Copyright (C) 2021 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Package generate implements the `syncthing generate` subcommand. +package generate + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "log" + "os" + + "github.com/syncthing/syncthing/cmd/syncthing/cmdutil" + "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/events" + "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/locations" + "github.com/syncthing/syncthing/lib/osutil" + "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/syncthing" +) + +type CLI struct { + cmdutil.CommonOptions + GUIUser string `placeholder:"STRING" help:"Specify new GUI authentication user name"` + GUIPassword string `placeholder:"STRING" help:"Specify new GUI authentication password (use - to read from standard input)"` +} + +func (c *CLI) Run() error { + log.SetFlags(0) + + if c.HideConsole { + osutil.HideConsole() + } + + if c.HomeDir != "" { + if c.ConfDir != "" { + return fmt.Errorf("--home must not be used together with --config") + } + c.ConfDir = c.HomeDir + } + if c.ConfDir == "" { + c.ConfDir = locations.GetBaseDir(locations.ConfigBaseDir) + } + + // Support reading the password from a pipe or similar + if c.GUIPassword == "-" { + reader := bufio.NewReader(os.Stdin) + password, _, err := reader.ReadLine() + if err != nil { + return fmt.Errorf("Failed reading GUI password: %w", err) + } + c.GUIPassword = string(password) + } + + if err := Generate(c.ConfDir, c.GUIUser, c.GUIPassword, c.NoDefaultFolder); err != nil { + return fmt.Errorf("Failed to generate config and keys: %w", err) + } + return nil +} + +func Generate(confDir, guiUser, guiPassword string, noDefaultFolder bool) error { + dir, err := fs.ExpandTilde(confDir) + if err != nil { + return err + } + + if err := syncthing.EnsureDir(dir, 0700); err != nil { + return err + } + locations.SetBaseDir(locations.ConfigBaseDir, dir) + + var myID protocol.DeviceID + certFile, keyFile := locations.Get(locations.CertFile), locations.Get(locations.KeyFile) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err == nil { + log.Println("WARNING: Key exists; will not overwrite.") + } else { + cert, err = syncthing.GenerateCertificate(certFile, keyFile) + if err != nil { + return fmt.Errorf("create certificate: %w", err) + } + } + myID = protocol.NewDeviceID(cert.Certificate[0]) + log.Println("Device ID:", myID) + + cfgFile := locations.Get(locations.ConfigFile) + var cfg config.Wrapper + if _, err := os.Stat(cfgFile); err == nil { + if guiUser == "" && guiPassword == "" { + log.Println("WARNING: Config exists; will not overwrite.") + return nil + } + + if cfg, _, err = config.Load(cfgFile, myID, events.NoopLogger); err != nil { + return fmt.Errorf("load config: %w", err) + } + } else { + if cfg, err = syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder); err != nil { + return fmt.Errorf("create config: %w", err) + } + } + + ctx, cancel := context.WithCancel(context.Background()) + go cfg.Serve(ctx) + defer cancel() + + var updateErr error + waiter, err := cfg.Modify(func(cfg *config.Configuration) { + updateErr = updateGUIAuthentication(&cfg.GUI, guiUser, guiPassword) + }) + if err != nil { + return fmt.Errorf("modify config: %w", err) + } + + waiter.Wait() + if updateErr != nil { + return updateErr + } + if err := cfg.Save(); err != nil { + return fmt.Errorf("save config: %w", err) + } + return nil +} + +func updateGUIAuthentication(guiCfg *config.GUIConfiguration, guiUser, guiPassword string) error { + if guiUser != "" && guiCfg.User != guiUser { + guiCfg.User = guiUser + log.Println("Updated GUI authentication user name:", guiUser) + } + + if guiPassword != "" && guiCfg.Password != guiPassword { + if err := guiCfg.HashAndSetPassword(guiPassword); err != nil { + return fmt.Errorf("Failed to set GUI authentication password: %w", err) + } + log.Println("Updated GUI authentication password.") + } + return nil +} diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 21a8fbe6bc5..127fcc54ff5 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -36,6 +36,7 @@ import ( "github.com/syncthing/syncthing/cmd/syncthing/cli" "github.com/syncthing/syncthing/cmd/syncthing/cmdutil" "github.com/syncthing/syncthing/cmd/syncthing/decrypt" + "github.com/syncthing/syncthing/cmd/syncthing/generate" "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/db" @@ -132,32 +133,30 @@ var ( // commands and options here are top level commands to syncthing. // Cli is just a placeholder for the help text (see main). var entrypoint struct { - Serve serveOptions `cmd:"" help:"Run Syncthing"` - Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"` - Cli struct{} `cmd:"" help:"Command line interface for Syncthing"` + Serve serveOptions `cmd:"" help:"Run Syncthing"` + Generate generate.CLI `cmd:"" help:"Generate key and config, then exit"` + Decrypt decrypt.CLI `cmd:"" help:"Decrypt or verify an encrypted folder"` + Cli struct{} `cmd:"" help:"Command line interface for Syncthing"` } // serveOptions are the options for the `syncthing serve` command. type serveOptions struct { - buildServeOptions + cmdutil.CommonOptions AllowNewerConfig bool `help:"Allow loading newer than current config version"` Audit bool `help:"Write events to audit file"` AuditFile string `name:"auditfile" placeholder:"PATH" help:"Specify audit file (use \"-\" for stdout, \"--\" for stderr)"` BrowserOnly bool `help:"Open GUI in browser"` - ConfDir string `name:"config" placeholder:"PATH" help:"Set configuration directory (config and keys)"` DataDir string `name:"data" placeholder:"PATH" help:"Set data directory (database and logs)"` DeviceID bool `help:"Show the device ID"` - GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` + GenerateDir string `name:"generate" placeholder:"PATH" help:"Generate key and config in specified dir, then exit"` //DEPRECATED: replaced by subcommand! GUIAddress string `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"` GUIAPIKey string `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"` - HomeDir string `name:"home" placeholder:"PATH" help:"Set configuration and data directory"` LogFile string `name:"logfile" default:"${logFile}" placeholder:"PATH" help:"Log file name (see below)"` LogFlags int `name:"logflags" default:"${logFlags}" placeholder:"BITS" help:"Select information in log line prefix (see below)"` LogMaxFiles int `placeholder:"N" default:"${logMaxFiles}" name:"log-max-old-files" help:"Number of old files to keep (zero to keep only current)"` LogMaxSize int `placeholder:"BYTES" default:"${logMaxSize}" help:"Maximum size of any file (zero to disable log rotation)"` NoBrowser bool `help:"Do not start browser"` NoRestart bool `env:"STNORESTART" help:"Do not restart Syncthing when exiting due to API/GUI command, upgrade, or crash"` - NoDefaultFolder bool `env:"STNODEFAULTFOLDER" help:"Don't create the \"default\" folder on first startup"` NoUpgrade bool `env:"STNOUPGRADE" help:"Disable automatic upgrades"` Paths bool `help:"Show configuration paths"` Paused bool `help:"Start with all devices and folders paused"` @@ -339,7 +338,7 @@ func (options serveOptions) Run() error { } if options.GenerateDir != "" { - if err := generate(options.GenerateDir, options.NoDefaultFolder); err != nil { + if err := generate.Generate(options.GenerateDir, "", "", options.NoDefaultFolder); err != nil { l.Warnln("Failed to generate config and keys:", err) os.Exit(svcutil.ExitError.AsInt()) } @@ -347,7 +346,7 @@ func (options serveOptions) Run() error { } // Ensure that our home directory exists. - if err := ensureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil { + if err := syncthing.EnsureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil { l.Warnln("Failure on home directory:", err) os.Exit(svcutil.ExitError.AsInt()) } @@ -413,8 +412,8 @@ func openGUI(myID protocol.DeviceID) error { if err != nil { return err } - if cfg.GUI().Enabled { - if err := openURL(cfg.GUI().URL()); err != nil { + if guiCfg := cfg.GUI(); guiCfg.Enabled { + if err := openURL(guiCfg.URL()); err != nil { return err } } else { @@ -423,46 +422,6 @@ func openGUI(myID protocol.DeviceID) error { return nil } -func generate(generateDir string, noDefaultFolder bool) error { - dir, err := fs.ExpandTilde(generateDir) - if err != nil { - return err - } - - if err := ensureDir(dir, 0700); err != nil { - return err - } - - var myID protocol.DeviceID - certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err == nil { - l.Warnln("Key exists; will not overwrite.") - } else { - cert, err = syncthing.GenerateCertificate(certFile, keyFile) - if err != nil { - return errors.Wrap(err, "create certificate") - } - } - myID = protocol.NewDeviceID(cert.Certificate[0]) - l.Infoln("Device ID:", myID) - - cfgFile := filepath.Join(dir, "config.xml") - if _, err := os.Stat(cfgFile); err == nil { - l.Warnln("Config exists; will not overwrite.") - return nil - } - cfg, err := syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder) - if err != nil { - return err - } - err = cfg.Save() - if err != nil { - return errors.Wrap(err, "save config") - } - return nil -} - func debugFacilities() string { facilities := l.Facilities() @@ -800,29 +759,6 @@ func resetDB() error { return os.RemoveAll(locations.Get(locations.Database)) } -func ensureDir(dir string, mode fs.FileMode) error { - fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) - err := fs.MkdirAll(".", mode) - if err != nil { - return err - } - - if fi, err := fs.Stat("."); err == nil { - // Apprently the stat may fail even though the mkdirall passed. If it - // does, we'll just assume things are in order and let other things - // fail (like loading or creating the config...). - currentMode := fi.Mode() & 0777 - if currentMode != mode { - err := fs.Chmod(".", mode) - // This can fail on crappy filesystems, nothing we can do about it. - if err != nil { - l.Warnln(err) - } - } - } - return nil -} - func standbyMonitor(app *syncthing.App, cfg config.Wrapper) { restartDelay := 60 * time.Second now := time.Now() diff --git a/lib/syncthing/utils.go b/lib/syncthing/utils.go index 468e94a045a..eecb563942f 100644 --- a/lib/syncthing/utils.go +++ b/lib/syncthing/utils.go @@ -24,6 +24,29 @@ import ( "github.com/syncthing/syncthing/lib/tlsutil" ) +func EnsureDir(dir string, mode fs.FileMode) error { + fs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + err := fs.MkdirAll(".", mode) + if err != nil { + return err + } + + if fi, err := fs.Stat("."); err == nil { + // Apprently the stat may fail even though the mkdirall passed. If it + // does, we'll just assume things are in order and let other things + // fail (like loading or creating the config...). + currentMode := fi.Mode() & 0777 + if currentMode != mode { + err := fs.Chmod(".", mode) + // This can fail on crappy filesystems, nothing we can do about it. + if err != nil { + l.Warnln(err) + } + } + } + return nil +} + func LoadOrGenerateCertificate(certFile, keyFile string) (tls.Certificate, error) { cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { From 8265dac1272b3690529bb2b532f82cda4fc43cc8 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Mon, 22 Nov 2021 08:29:44 +0100 Subject: [PATCH 013/220] lib/nat: Fix race condition in Mapping (#8042) The locking protocol in nat.Mapping was racy: * Mapping.addressMap RLock'd, but then returned a map shared between caller and Mapping, so the lock didn't do anything. * Operations inside Service.{verifyExistingMappings,acquireNewMappings} would lock the map for every update, but that means callers to Mapping.ExternalAddresses can be looping over the map while the Service methods are concurrently modifying it. When the Go runtime detects that happening, it panics. * Mapping.expires was read and updated without locking. The Service methods now lock the map once and release the lock only when done. Also, subscribers no longer get the added and removed addresses, because none of them were using the information. This was changed for a previous attempt to retain the fine-grained locking and not reverted because it simplifies the code. --- cmd/strelaysrv/main.go | 2 +- lib/connections/tcp_listen.go | 2 +- lib/nat/service.go | 66 ++++++++++++++++------------------- lib/nat/structs.go | 34 ++++++------------ 4 files changed, 43 insertions(+), 61 deletions(-) diff --git a/cmd/strelaysrv/main.go b/cmd/strelaysrv/main.go index fea34ccd440..507b3e6c01b 100644 --- a/cmd/strelaysrv/main.go +++ b/cmd/strelaysrv/main.go @@ -200,7 +200,7 @@ func main() { go natSvc.Serve(ctx) defer cancel() found := make(chan struct{}) - mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) { + mapping.OnChanged(func() { select { case found <- struct{}{}: default: diff --git a/lib/connections/tcp_listen.go b/lib/connections/tcp_listen.go index 54b833a7723..13a0713343b 100644 --- a/lib/connections/tcp_listen.go +++ b/lib/connections/tcp_listen.go @@ -76,7 +76,7 @@ func (t *tcpListener) serve(ctx context.Context) error { defer l.Infof("TCP listener (%v) shutting down", tcaddr) mapping := t.natService.NewMapping(nat.TCP, tcaddr.IP, tcaddr.Port) - mapping.OnChanged(func(_ *nat.Mapping, _, _ []nat.Address) { + mapping.OnChanged(func() { t.notifyAddressesChanged(t) }) // Should be called after t.mapping is nil'ed out. diff --git a/lib/nat/service.go b/lib/nat/service.go index 4c0269c48ec..0785879616a 100644 --- a/lib/nat/service.go +++ b/lib/nat/service.go @@ -125,11 +125,15 @@ func (s *Service) process(ctx context.Context) (int, time.Duration) { s.mut.RLock() for _, mapping := range s.mappings { - if mapping.expires.Before(time.Now()) { + mapping.mut.RLock() + expires := mapping.expires + mapping.mut.RUnlock() + + if expires.Before(time.Now()) { toRenew = append(toRenew, mapping) } else { toUpdate = append(toUpdate, mapping) - mappingRenewIn := time.Until(mapping.expires) + mappingRenewIn := time.Until(expires) if mappingRenewIn < renewIn { renewIn = mappingRenewIn } @@ -206,41 +210,36 @@ func (s *Service) RemoveMapping(mapping *Mapping) { // Optionally takes renew flag which indicates whether or not we should renew // mappings with existing natds func (s *Service) updateMapping(ctx context.Context, mapping *Mapping, nats map[string]Device, renew bool) { - var added, removed []Address - renewalTime := time.Duration(s.cfg.Options().NATRenewalM) * time.Minute - mapping.expires = time.Now().Add(renewalTime) - newAdded, newRemoved := s.verifyExistingMappings(ctx, mapping, nats, renew) - added = append(added, newAdded...) - removed = append(removed, newRemoved...) + mapping.mut.Lock() - newAdded, newRemoved = s.acquireNewMappings(ctx, mapping, nats) - added = append(added, newAdded...) - removed = append(removed, newRemoved...) + mapping.expires = time.Now().Add(renewalTime) + change := s.verifyExistingLocked(ctx, mapping, nats, renew) + add := s.acquireNewLocked(ctx, mapping, nats) - if len(added) > 0 || len(removed) > 0 { - mapping.notify(added, removed) + mapping.mut.Unlock() + + if change || add { + mapping.notify() } } -func (s *Service) verifyExistingMappings(ctx context.Context, mapping *Mapping, nats map[string]Device, renew bool) ([]Address, []Address) { - var added, removed []Address - +func (s *Service) verifyExistingLocked(ctx context.Context, mapping *Mapping, nats map[string]Device, renew bool) (change bool) { leaseTime := time.Duration(s.cfg.Options().NATLeaseM) * time.Minute - for id, address := range mapping.addressMap() { + for id, address := range mapping.extAddresses { select { case <-ctx.Done(): - return nil, nil + return false default: } // Delete addresses for NATDevice's that do not exist anymore nat, ok := nats[id] if !ok { - mapping.removeAddress(id) - removed = append(removed, address) + mapping.removeAddressLocked(id) + change = true continue } else if renew { // Only perform renewals on the nat's that have the right local IP @@ -256,35 +255,32 @@ func (s *Service) verifyExistingMappings(ctx context.Context, mapping *Mapping, addr, err := s.tryNATDevice(ctx, nat, mapping.address.Port, address.Port, leaseTime) if err != nil { l.Debugf("Failed to renew %s -> mapping on %s", mapping, address, id) - mapping.removeAddress(id) - removed = append(removed, address) + mapping.removeAddressLocked(id) + change = true continue } l.Debugf("Renewed %s -> %s mapping on %s", mapping, address, id) if !addr.Equal(address) { - mapping.removeAddress(id) - mapping.setAddress(id, addr) - removed = append(removed, address) - added = append(added, address) + mapping.removeAddressLocked(id) + mapping.setAddressLocked(id, addr) + change = true } } } - return added, removed + return change } -func (s *Service) acquireNewMappings(ctx context.Context, mapping *Mapping, nats map[string]Device) ([]Address, []Address) { - var added, removed []Address - +func (s *Service) acquireNewLocked(ctx context.Context, mapping *Mapping, nats map[string]Device) (change bool) { leaseTime := time.Duration(s.cfg.Options().NATLeaseM) * time.Minute - addrMap := mapping.addressMap() + addrMap := mapping.extAddresses for id, nat := range nats { select { case <-ctx.Done(): - return nil, nil + return false default: } @@ -310,11 +306,11 @@ func (s *Service) acquireNewMappings(ctx context.Context, mapping *Mapping, nats l.Debugf("Acquired %s -> %s mapping on %s", mapping, addr, id) - mapping.setAddress(id, addr) - added = append(added, addr) + mapping.setAddressLocked(id, addr) + change = true } - return added, removed + return change } // tryNATDevice tries to acquire a port mapping for the given internal address to diff --git a/lib/nat/structs.go b/lib/nat/structs.go index 32dae4a5f2c..29be5b8d154 100644 --- a/lib/nat/structs.go +++ b/lib/nat/structs.go @@ -14,7 +14,7 @@ import ( "github.com/syncthing/syncthing/lib/sync" ) -type MappingChangeSubscriber func(*Mapping, []Address, []Address) +type MappingChangeSubscriber func() type Mapping struct { protocol Protocol @@ -26,55 +26,41 @@ type Mapping struct { mut sync.RWMutex } -func (m *Mapping) setAddress(id string, address Address) { - m.mut.Lock() - if existing, ok := m.extAddresses[id]; !ok || !existing.Equal(address) { - l.Infof("New NAT port mapping: external %s address %s to local address %s.", m.protocol, address, m.address) - m.extAddresses[id] = address - } - m.mut.Unlock() +func (m *Mapping) setAddressLocked(id string, address Address) { + l.Infof("New NAT port mapping: external %s address %s to local address %s.", m.protocol, address, m.address) + m.extAddresses[id] = address } -func (m *Mapping) removeAddress(id string) { - m.mut.Lock() +func (m *Mapping) removeAddressLocked(id string) { addr, ok := m.extAddresses[id] if ok { l.Infof("Removing NAT port mapping: external %s address %s, NAT %s is no longer available.", m.protocol, addr, id) delete(m.extAddresses, id) } - m.mut.Unlock() } func (m *Mapping) clearAddresses() { m.mut.Lock() - var removed []Address + change := len(m.extAddresses) > 0 for id, addr := range m.extAddresses { l.Debugf("Clearing mapping %s: ID: %s Address: %s", m, id, addr) - removed = append(removed, addr) delete(m.extAddresses, id) } m.expires = time.Time{} m.mut.Unlock() - if len(removed) > 0 { - m.notify(nil, removed) + if change { + m.notify() } } -func (m *Mapping) notify(added, removed []Address) { +func (m *Mapping) notify() { m.mut.RLock() for _, subscriber := range m.subscribers { - subscriber(m, added, removed) + subscriber() } m.mut.RUnlock() } -func (m *Mapping) addressMap() map[string]Address { - m.mut.RLock() - addrMap := m.extAddresses - m.mut.RUnlock() - return addrMap -} - func (m *Mapping) Protocol() Protocol { return m.protocol } From e2288fe441820794c983c3036829fd52c37ae1ea Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 22 Nov 2021 08:31:03 +0100 Subject: [PATCH 014/220] lib/relay: Send SNI when the address is a host name (fixes #8014) (#8015) --- lib/relay/client/static.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/relay/client/static.go b/lib/relay/client/static.go index 1f5cdb18547..a49768b84e9 100644 --- a/lib/relay/client/static.go +++ b/lib/relay/client/static.go @@ -141,7 +141,17 @@ func (c *staticClient) connect(ctx context.Context) error { return err } - conn := tls.Client(tcpConn, c.config) + // Copy the TLS config and set the server name we're connecting to. In + // many cases this will be an IP address, in which case it's a no-op. In + // other cases it will be a hostname, which will cause the TLS stack to + // send SNI. + cfg := c.config + if host, _, err := net.SplitHostPort(c.uri.Host); err == nil { + cfg = cfg.Clone() + cfg.ServerName = host + } + + conn := tls.Client(tcpConn, cfg) if err := conn.SetDeadline(time.Now().Add(c.connectTimeout)); err != nil { conn.Close() From bf89bffb0b844cfafc8a38467870157e52e6f198 Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Mon, 22 Nov 2021 08:45:29 +0100 Subject: [PATCH 015/220] lib/config: Decouple VerifyConfiguration from Committer (#7939) ... and remove 8/10 implementations, which were no-ops. This saves code and time copying configurations. --- lib/api/api.go | 2 ++ lib/config/wrapper.go | 23 ++++++++++++++++------- lib/connections/limiter.go | 4 ---- lib/connections/service.go | 4 ---- lib/discover/manager.go | 4 ---- lib/model/model.go | 2 ++ lib/model/progressemitter.go | 5 ----- lib/nat/service.go | 4 ---- lib/ur/failurereporting.go | 4 ---- lib/ur/usage_report.go | 4 ---- lib/watchaggregator/aggregator.go | 4 ---- 11 files changed, 20 insertions(+), 40 deletions(-) diff --git a/lib/api/api.go b/lib/api/api.go index 360e94212c6..6ba4235f347 100644 --- a/lib/api/api.go +++ b/lib/api/api.go @@ -94,6 +94,8 @@ type service struct { systemLog logger.Recorder } +var _ config.Verifier = &service{} + type Service interface { suture.Service config.Committer diff --git a/lib/config/wrapper.go b/lib/config/wrapper.go index 3482b135a81..4d5c8f93ccf 100644 --- a/lib/config/wrapper.go +++ b/lib/config/wrapper.go @@ -31,14 +31,14 @@ const ( var errTooManyModifications = errors.New("too many concurrent config modifications") -// The Committer interface is implemented by objects that need to know about -// or have a say in configuration changes. +// The Committer and Verifier interfaces are implemented by objects +// that need to know about or have a say in configuration changes. // // When the configuration is about to be changed, VerifyConfiguration() is -// called for each subscribing object, with the old and new configuration. A -// nil error is returned if the new configuration is acceptable (i.e. does not -// contain any errors that would prevent it from being a valid config). -// Otherwise an error describing the problem is returned. +// called for each subscribing object that implements it, with copies of the +// old and new configuration. A nil error is returned if the new configuration +// is acceptable (i.e., does not contain any errors that would prevent it from +// being a valid config). Otherwise an error describing the problem is returned. // // If any subscriber returns an error from VerifyConfiguration(), the // configuration change is not committed and an error is returned to whoever @@ -55,11 +55,16 @@ var errTooManyModifications = errors.New("too many concurrent config modificatio // configuration (e.g. calling Wrapper.SetFolder), that are also acquired in any // methods of the Committer interface. type Committer interface { - VerifyConfiguration(from, to Configuration) error CommitConfiguration(from, to Configuration) (handled bool) String() string } +// A Verifier can determine if a new configuration is acceptable. +// See the description for Committer, above. +type Verifier interface { + VerifyConfiguration(from, to Configuration) error +} + // Waiter allows to wait for the given config operation to complete. type Waiter interface { Wait() @@ -300,6 +305,10 @@ func (w *wrapper) replaceLocked(to Configuration) (Waiter, error) { } for _, sub := range w.subs { + sub, ok := sub.(Verifier) + if !ok { + continue + } l.Debugln(sub, "verifying configuration") if err := sub.VerifyConfiguration(from.Copy(), to.Copy()); err != nil { l.Debugln(sub, "rejected config:", err) diff --git a/lib/connections/limiter.go b/lib/connections/limiter.go index eac63d70e1b..66331c221f7 100644 --- a/lib/connections/limiter.go +++ b/lib/connections/limiter.go @@ -122,10 +122,6 @@ func (lim *limiter) processDevicesConfigurationLocked(from, to config.Configurat } } -func (lim *limiter) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - func (lim *limiter) CommitConfiguration(from, to config.Configuration) bool { // to ensure atomic update of configuration lim.mu.Lock() diff --git a/lib/connections/service.go b/lib/connections/service.go index 3848b261685..24a6f142045 100644 --- a/lib/connections/service.go +++ b/lib/connections/service.go @@ -706,10 +706,6 @@ func (s *service) logListenAddressesChangedEvent(l ListenerAddresses) { }) } -func (s *service) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - func (s *service) CommitConfiguration(from, to config.Configuration) bool { newDevices := make(map[protocol.DeviceID]bool, len(to.Devices)) for _, dev := range to.Devices { diff --git a/lib/discover/manager.go b/lib/discover/manager.go index ec43e889cbd..686d673104c 100644 --- a/lib/discover/manager.go +++ b/lib/discover/manager.go @@ -227,10 +227,6 @@ func (m *manager) Cache() map[protocol.DeviceID]CacheEntry { return res } -func (m *manager) VerifyConfiguration(_, _ config.Configuration) error { - return nil -} - func (m *manager) CommitConfiguration(_, to config.Configuration) (handled bool) { m.mut.Lock() defer m.mut.Unlock() diff --git a/lib/model/model.go b/lib/model/model.go index 28a3a5469e8..e722a90cc19 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -168,6 +168,8 @@ type model struct { foldersRunning int32 } +var _ config.Verifier = &model{} + type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *util.Semaphore) service var ( diff --git a/lib/model/progressemitter.go b/lib/model/progressemitter.go index 56bf92e27ce..fab9e72c96a 100644 --- a/lib/model/progressemitter.go +++ b/lib/model/progressemitter.go @@ -212,11 +212,6 @@ func (t *ProgressEmitter) computeProgressUpdates() []progressUpdate { return progressUpdates } -// VerifyConfiguration implements the config.Committer interface -func (t *ProgressEmitter) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - // CommitConfiguration implements the config.Committer interface func (t *ProgressEmitter) CommitConfiguration(_, to config.Configuration) bool { t.mut.Lock() diff --git a/lib/nat/service.go b/lib/nat/service.go index 0785879616a..5e2f1c704e1 100644 --- a/lib/nat/service.go +++ b/lib/nat/service.go @@ -45,10 +45,6 @@ func NewService(id protocol.DeviceID, cfg config.Wrapper) *Service { return s } -func (s *Service) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - func (s *Service) CommitConfiguration(from, to config.Configuration) bool { s.mut.Lock() if !s.enabled && to.Options.NATEnabled { diff --git a/lib/ur/failurereporting.go b/lib/ur/failurereporting.go index 899bd0e303c..f8d4d88507b 100644 --- a/lib/ur/failurereporting.go +++ b/lib/ur/failurereporting.go @@ -188,10 +188,6 @@ func (h *failureHandler) addReport(data FailureData, evTime time.Time) { } } -func (h *failureHandler) VerifyConfiguration(_, _ config.Configuration) error { - return nil -} - func (h *failureHandler) CommitConfiguration(from, to config.Configuration) bool { if from.Options.CREnabled != to.Options.CREnabled || from.Options.CRURL != to.Options.CRURL { h.optsChan <- to.Options diff --git a/lib/ur/usage_report.go b/lib/ur/usage_report.go index a2d2b0923a5..f6537f20f8a 100644 --- a/lib/ur/usage_report.go +++ b/lib/ur/usage_report.go @@ -390,10 +390,6 @@ func (s *Service) Serve(ctx context.Context) error { } } -func (s *Service) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - func (s *Service) CommitConfiguration(from, to config.Configuration) bool { if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL { select { diff --git a/lib/watchaggregator/aggregator.go b/lib/watchaggregator/aggregator.go index 93a38c4bad4..5c176fde8c5 100644 --- a/lib/watchaggregator/aggregator.go +++ b/lib/watchaggregator/aggregator.go @@ -419,10 +419,6 @@ func (a *aggregator) String() string { return fmt.Sprintf("aggregator/%s:", a.folderCfg.Description()) } -func (a *aggregator) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - func (a *aggregator) CommitConfiguration(from, to config.Configuration) bool { for _, folderCfg := range to.Folders { if folderCfg.ID == a.folderID { From 4b750b6dc3dc9904c6046d1c6370b57e1710cb4b Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 22 Nov 2021 08:59:47 +0100 Subject: [PATCH 016/220] all: Remove usage of deprecated io/ioutil (#7971) As of Go 1.16 io/ioutil is deprecated. This replaces usage with the corresponding functions in package os and package io. --- build.go | 13 +++++------ cmd/stcrashreceiver/main.go | 3 +-- cmd/stcrashreceiver/sentry.go | 4 ++-- cmd/stcrashreceiver/sentry_test.go | 4 ++-- cmd/stcrashreceiver/sourcecodeloader.go | 4 ++-- cmd/stcrashreceiver/stcrashreceiver.go | 5 ++-- cmd/stcrashreceiver/util.go | 4 ++-- cmd/strelaypoolsrv/main.go | 7 +++--- cmd/strelaysrv/pool.go | 4 ++-- cmd/stsigtool/main.go | 7 +++--- cmd/syncthing/cli/main.go | 4 ++-- cmd/syncthing/cli/utils.go | 4 ++-- cmd/syncthing/crash_reporting.go | 3 +-- cmd/syncthing/decrypt/decrypt.go | 4 ++-- cmd/syncthing/main.go | 3 +-- cmd/syncthing/monitor_test.go | 7 +++--- cmd/ursrv/main.go | 5 ++-- lib/api/api.go | 15 ++++++------ lib/api/api_test.go | 13 +++++------ lib/api/auto/auto_test.go | 4 ++-- lib/api/confighandler.go | 3 +-- lib/assets/assets_test.go | 5 ++-- lib/config/config.go | 3 +-- lib/config/config_test.go | 9 ++++--- lib/discover/global.go | 3 +-- lib/discover/global_test.go | 4 ++-- lib/fs/basicfs_test.go | 3 +-- lib/fs/basicfs_watch_test.go | 3 +-- lib/fs/fakefs.go | 3 +-- lib/fs/fakefs_test.go | 13 +++++------ lib/fs/filesystem_copy_range_test.go | 5 ++-- lib/fs/mtimefs_test.go | 17 +++++++------- lib/ignore/ignore_test.go | 13 +++++------ lib/logger/logger.go | 3 +-- lib/logger/logger_test.go | 6 ++--- lib/model/folder_sendrecv_test.go | 5 ++-- lib/model/model_test.go | 9 ++++--- lib/model/requests_test.go | 5 ++-- lib/model/testutils_test.go | 5 ++-- lib/nat/structs_test.go | 3 +-- lib/osutil/atomic.go | 2 +- lib/osutil/atomic_test.go | 11 ++++----- lib/osutil/atomic_unix_test.go | 3 +-- lib/osutil/osutil_test.go | 8 +++---- lib/osutil/traversessymlink_test.go | 5 ++-- lib/protocol/protocol_test.go | 6 ++--- lib/rc/rc.go | 3 +-- lib/scanner/walk_test.go | 7 +++--- lib/syncthing/syncthing_test.go | 3 +-- lib/syncthing/utils.go | 5 ++-- lib/upgrade/upgrade_supported.go | 7 +++--- lib/upnp/upnp.go | 4 ++-- lib/versioner/external_test.go | 3 +-- lib/versioner/simple_test.go | 4 ++-- lib/versioner/staggered_test.go | 6 ++--- lib/versioner/trashcan_test.go | 9 +++---- lib/versioner/versioner_test.go | 3 +-- lib/weakhash/weakhash_test.go | 3 +-- script/authors.go | 6 ++--- script/commit-msg.go | 3 +-- script/genassets.go | 3 +-- script/prune_mocks.go | 3 +-- script/transifexdl.go | 4 ++-- test/conflict_test.go | 31 ++++++++++++------------- test/delay_scan_test.go | 4 ++-- test/http_test.go | 9 +++---- test/ignore_test.go | 5 ++-- test/override_test.go | 16 ++++++------- test/parallell_scan_test.go | 4 ++-- test/reset_test.go | 3 +-- test/util.go | 5 ++-- 71 files changed, 190 insertions(+), 235 deletions(-) diff --git a/build.go b/build.go index 791ca9a41e1..2a480a5464d 100644 --- a/build.go +++ b/build.go @@ -20,7 +20,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "os" "os/exec" @@ -723,7 +722,7 @@ func shouldBuildSyso(dir string) (string, error) { } jsonPath := filepath.Join(dir, "versioninfo.json") - err = ioutil.WriteFile(jsonPath, bs, 0644) + err = os.WriteFile(jsonPath, bs, 0644) if err != nil { return "", errors.New("failed to create " + jsonPath + ": " + err.Error()) } @@ -762,12 +761,12 @@ func shouldCleanupSyso(sysoFilePath string) { // exists. The permission bits are copied as well. If dst already exists and // the contents are identical to src the modification time is not updated. func copyFile(src, dst string, perm os.FileMode) error { - in, err := ioutil.ReadFile(src) + in, err := os.ReadFile(src) if err != nil { return err } - out, err := ioutil.ReadFile(dst) + out, err := os.ReadFile(dst) if err != nil { // The destination probably doesn't exist, we should create // it. @@ -783,7 +782,7 @@ func copyFile(src, dst string, perm os.FileMode) error { copy: os.MkdirAll(filepath.Dir(dst), 0777) - if err := ioutil.WriteFile(dst, in, perm); err != nil { + if err := os.WriteFile(dst, in, perm); err != nil { return err } @@ -958,7 +957,7 @@ func rmr(paths ...string) { } func getReleaseVersion() (string, error) { - bs, err := ioutil.ReadFile("RELEASE") + bs, err := os.ReadFile("RELEASE") if err != nil { return "", err } @@ -1290,7 +1289,7 @@ func zipFile(out string, files []archiveFile) { if strings.HasSuffix(f.dst, ".txt") { // Text file. Read it and convert line endings. - bs, err := ioutil.ReadAll(sf) + bs, err := io.ReadAll(sf) if err != nil { log.Fatal(err) } diff --git a/cmd/stcrashreceiver/main.go b/cmd/stcrashreceiver/main.go index 31ee52ae654..c6a1871614f 100644 --- a/cmd/stcrashreceiver/main.go +++ b/cmd/stcrashreceiver/main.go @@ -17,7 +17,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -58,7 +57,7 @@ func main() { func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { lr := io.LimitReader(req.Body, maxRequestSize) - bs, err := ioutil.ReadAll(lr) + bs, err := io.ReadAll(lr) req.Body.Close() if err != nil { http.Error(w, err.Error(), 500) diff --git a/cmd/stcrashreceiver/sentry.go b/cmd/stcrashreceiver/sentry.go index 5aa8a7c28d3..e49eba88824 100644 --- a/cmd/stcrashreceiver/sentry.go +++ b/cmd/stcrashreceiver/sentry.go @@ -9,7 +9,7 @@ package main import ( "bytes" "errors" - "io/ioutil" + "io" "regexp" "strings" "sync" @@ -93,7 +93,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { } r := bytes.NewReader(report) - ctx, err := stack.ParseDump(r, ioutil.Discard, false) + ctx, err := stack.ParseDump(r, io.Discard, false) if err != nil { return nil, err } diff --git a/cmd/stcrashreceiver/sentry_test.go b/cmd/stcrashreceiver/sentry_test.go index fbfd2c42132..e052197c913 100644 --- a/cmd/stcrashreceiver/sentry_test.go +++ b/cmd/stcrashreceiver/sentry_test.go @@ -8,7 +8,7 @@ package main import ( "fmt" - "io/ioutil" + "os" "testing" ) @@ -59,7 +59,7 @@ func TestParseVersion(t *testing.T) { } func TestParseReport(t *testing.T) { - bs, err := ioutil.ReadFile("_testdata/panic.log") + bs, err := os.ReadFile("_testdata/panic.log") if err != nil { t.Fatal(err) } diff --git a/cmd/stcrashreceiver/sourcecodeloader.go b/cmd/stcrashreceiver/sourcecodeloader.go index 1da310d3ce5..e51d8335bc6 100644 --- a/cmd/stcrashreceiver/sourcecodeloader.go +++ b/cmd/stcrashreceiver/sourcecodeloader.go @@ -9,7 +9,7 @@ package main import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "path/filepath" "strings" @@ -80,7 +80,7 @@ func (l *githubSourceCodeLoader) Load(filename string, line, context int) ([][]b fmt.Println("Loading source:", resp.Status) return nil, 0 } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) _ = resp.Body.Close() if err != nil { fmt.Println("Loading source:", err.Error()) diff --git a/cmd/stcrashreceiver/stcrashreceiver.go b/cmd/stcrashreceiver/stcrashreceiver.go index 9a39296c12e..f6f988f3e08 100644 --- a/cmd/stcrashreceiver/stcrashreceiver.go +++ b/cmd/stcrashreceiver/stcrashreceiver.go @@ -10,7 +10,6 @@ import ( "bytes" "compress/gzip" "io" - "io/ioutil" "log" "net/http" "os" @@ -96,7 +95,7 @@ func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWrite // Read at most maxRequestSize of report data. log.Println("Receiving report", reportID) lr := io.LimitReader(req.Body, maxRequestSize) - bs, err := ioutil.ReadAll(lr) + bs, err := io.ReadAll(lr) if err != nil { log.Println("Reading report:", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -110,7 +109,7 @@ func (r *crashReceiver) servePut(reportID, fullPath string, w http.ResponseWrite gw.Close() // Create an output file with the compressed report - err = ioutil.WriteFile(fullPath, buf.Bytes(), 0644) + err = os.WriteFile(fullPath, buf.Bytes(), 0644) if err != nil { log.Println("Saving report:", err) http.Error(w, "Internal server error", http.StatusInternalServerError) diff --git a/cmd/stcrashreceiver/util.go b/cmd/stcrashreceiver/util.go index ac164a1c611..ecd0fde914f 100644 --- a/cmd/stcrashreceiver/util.go +++ b/cmd/stcrashreceiver/util.go @@ -10,9 +10,9 @@ import ( "bytes" "compress/gzip" "fmt" - "io/ioutil" "net" "net/http" + "os" "path/filepath" "time" @@ -53,5 +53,5 @@ func compressAndWrite(bs []byte, fullPath string) error { gw.Close() // Create an output file with the compressed report - return ioutil.WriteFile(fullPath, buf.Bytes(), 0644) + return os.WriteFile(fullPath, buf.Bytes(), 0644) } diff --git a/cmd/strelaypoolsrv/main.go b/cmd/strelaypoolsrv/main.go index a409b3c686e..a0364a81c55 100644 --- a/cmd/strelaypoolsrv/main.go +++ b/cmd/strelaypoolsrv/main.go @@ -11,7 +11,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -560,7 +559,7 @@ func limit(addr string, cache *lru.Cache, lock sync.Mutex, intv time.Duration, b } func loadRelays(file string) []*relay { - content, err := ioutil.ReadFile(file) + content, err := os.ReadFile(file) if err != nil { log.Println("Failed to load relays: " + err.Error()) return nil @@ -598,11 +597,11 @@ func saveRelays(file string, relays []*relay) error { for _, relay := range relays { content += relay.uri.String() + "\n" } - return ioutil.WriteFile(file, []byte(content), 0777) + return os.WriteFile(file, []byte(content), 0777) } func createTestCertificate() tls.Certificate { - tmpDir, err := ioutil.TempDir("", "relaypoolsrv") + tmpDir, err := os.MkdirTemp("", "relaypoolsrv") if err != nil { log.Fatal(err) } diff --git a/cmd/strelaysrv/pool.go b/cmd/strelaysrv/pool.go index 9ddde16c58d..8aa48c935b7 100644 --- a/cmd/strelaysrv/pool.go +++ b/cmd/strelaysrv/pool.go @@ -6,7 +6,7 @@ import ( "bytes" "crypto/tls" "encoding/json" - "io/ioutil" + "io" "log" "net/http" "net/url" @@ -56,7 +56,7 @@ func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certifi continue } - bs, err := ioutil.ReadAll(resp.Body) + bs, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { log.Printf("Error joining pool %v: reading response: %v", pool, err) diff --git a/cmd/stsigtool/main.go b/cmd/stsigtool/main.go index 0217f17e801..ab4c46b3e83 100644 --- a/cmd/stsigtool/main.go +++ b/cmd/stsigtool/main.go @@ -9,7 +9,6 @@ package main import ( "flag" "io" - "io/ioutil" "log" "os" @@ -69,7 +68,7 @@ func gen() { } func sign(keyname, dataname string) { - privkey, err := ioutil.ReadFile(keyname) + privkey, err := os.ReadFile(keyname) if err != nil { log.Fatal(err) } @@ -95,7 +94,7 @@ func sign(keyname, dataname string) { } func verifyWithFile(signame, dataname, keyname string) { - pubkey, err := ioutil.ReadFile(keyname) + pubkey, err := os.ReadFile(keyname) if err != nil { log.Fatal(err) } @@ -103,7 +102,7 @@ func verifyWithFile(signame, dataname, keyname string) { } func verifyWithKey(signame, dataname string, pubkey []byte) { - sig, err := ioutil.ReadFile(signame) + sig, err := os.ReadFile(signame) if err != nil { log.Fatal(err) } diff --git a/cmd/syncthing/cli/main.go b/cmd/syncthing/cli/main.go index 8e08b92ef5a..b6599c893b1 100644 --- a/cmd/syncthing/cli/main.go +++ b/cmd/syncthing/cli/main.go @@ -9,7 +9,7 @@ package cli import ( "bufio" "fmt" - "io/ioutil" + "io" "os" "strings" @@ -151,7 +151,7 @@ func parseFlags(c *preCli) error { } } // We don't want kong to print anything nor os.Exit (e.g. on -h) - parser, err := kong.New(c, kong.Writers(ioutil.Discard, ioutil.Discard), kong.Exit(func(int) {})) + parser, err := kong.New(c, kong.Writers(io.Discard, io.Discard), kong.Exit(func(int) {})) if err != nil { return err } diff --git a/cmd/syncthing/cli/utils.go b/cmd/syncthing/cli/utils.go index 17bde14379c..c495458e10c 100644 --- a/cmd/syncthing/cli/utils.go +++ b/cmd/syncthing/cli/utils.go @@ -10,7 +10,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "mime" "net/http" "os" @@ -23,7 +23,7 @@ import ( ) func responseToBArray(response *http.Response) ([]byte, error) { - bytes, err := ioutil.ReadAll(response.Body) + bytes, err := io.ReadAll(response.Body) if err != nil { return nil, err } diff --git a/cmd/syncthing/crash_reporting.go b/cmd/syncthing/crash_reporting.go index 828695f1aad..27896ff8a61 100644 --- a/cmd/syncthing/crash_reporting.go +++ b/cmd/syncthing/crash_reporting.go @@ -10,7 +10,6 @@ import ( "bytes" "context" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -62,7 +61,7 @@ func uploadPanicLogs(ctx context.Context, urlBase, dir string) { // the log contents. A HEAD request is made to see if the log has already // been reported. If not, a PUT is made with the log contents. func uploadPanicLog(ctx context.Context, urlBase, file string) error { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return err } diff --git a/cmd/syncthing/decrypt/decrypt.go b/cmd/syncthing/decrypt/decrypt.go index 119583dbd70..bb7651e7257 100644 --- a/cmd/syncthing/decrypt/decrypt.go +++ b/cmd/syncthing/decrypt/decrypt.go @@ -12,8 +12,8 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" + "os" "path/filepath" "github.com/syncthing/syncthing/lib/config" @@ -112,7 +112,7 @@ func (c *CLI) withContinue(err error) error { // error. func (c *CLI) getFolderID() (string, error) { tokenPath := filepath.Join(c.Path, c.TokenPath) - bs, err := ioutil.ReadFile(tokenPath) + bs, err := os.ReadFile(tokenPath) if err != nil { return "", fmt.Errorf("reading folder token: %w", err) } diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 127fcc54ff5..3e3604d60a8 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -12,7 +12,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "log" "net/http" _ "net/http/pprof" // Need to import this to support STPROFILER. @@ -500,7 +499,7 @@ func upgradeViaRest() error { return err } if resp.StatusCode != 200 { - bs, err := ioutil.ReadAll(resp.Body) + bs, err := io.ReadAll(resp.Body) defer resp.Body.Close() if err != nil { return err diff --git a/cmd/syncthing/monitor_test.go b/cmd/syncthing/monitor_test.go index 345a9a06ceb..6ad8e4ee2d1 100644 --- a/cmd/syncthing/monitor_test.go +++ b/cmd/syncthing/monitor_test.go @@ -8,7 +8,6 @@ package main import ( "io" - "io/ioutil" "os" "path/filepath" "testing" @@ -18,7 +17,7 @@ import ( func TestRotatedFile(t *testing.T) { // Verify that log rotation happens. - dir, err := ioutil.TempDir("", "syncthing") + dir, err := os.MkdirTemp("", "syncthing") if err != nil { t.Fatal(err) } @@ -179,7 +178,7 @@ func TestAutoClosedFile(t *testing.T) { } // The file should have both writes in it. - bs, err := ioutil.ReadFile(file) + bs, err := os.ReadFile(file) if err != nil { t.Fatal(err) } @@ -199,7 +198,7 @@ func TestAutoClosedFile(t *testing.T) { } // It should now contain three writes, as the file is always opened for appending - bs, err = ioutil.ReadFile(file) + bs, err = os.ReadFile(file) if err != nil { t.Fatal(err) } diff --git a/cmd/ursrv/main.go b/cmd/ursrv/main.go index 6cbf4429932..0d24ee1be94 100644 --- a/cmd/ursrv/main.go +++ b/cmd/ursrv/main.go @@ -13,7 +13,6 @@ import ( "encoding/json" "html/template" "io" - "io/ioutil" "log" "net" "net/http" @@ -162,7 +161,7 @@ func main() { if err != nil { log.Fatalln("template:", err) } - bs, err := ioutil.ReadAll(fd) + bs, err := io.ReadAll(fd) if err != nil { log.Fatalln("template:", err) } @@ -324,7 +323,7 @@ func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) { rep.Address = addr lr := &io.LimitedReader{R: r.Body, N: 40 * 1024} - bs, _ := ioutil.ReadAll(lr) + bs, _ := io.ReadAll(lr) if err := json.Unmarshal(bs, &rep); err != nil { log.Println("decode:", err) if debug { diff --git a/lib/api/api.go b/lib/api/api.go index 6ba4235f347..1e8c4747b85 100644 --- a/lib/api/api.go +++ b/lib/api/api.go @@ -15,7 +15,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -382,7 +381,7 @@ func (s *service) Serve(ctx context.Context) error { ReadTimeout: 15 * time.Second, // Prevent the HTTP server from logging stuff on its own. The things we // care about we log ourselves from the handlers. - ErrorLog: log.New(ioutil.Discard, "", 0), + ErrorLog: log.New(io.Discard, "", 0), } l.Infoln("GUI and API listening on", listener.Addr()) @@ -1098,7 +1097,7 @@ func (s *service) getSystemError(w http.ResponseWriter, r *http.Request) { } func (s *service) postSystemError(w http.ResponseWriter, r *http.Request) { - bs, _ := ioutil.ReadAll(r.Body) + bs, _ := io.ReadAll(r.Body) r.Body.Close() l.Warnln(string(bs)) } @@ -1165,7 +1164,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) { // Panic files if panicFiles, err := filepath.Glob(filepath.Join(locations.GetBaseDir(locations.ConfigBaseDir), "panic*")); err == nil { for _, f := range panicFiles { - if panicFile, err := ioutil.ReadFile(f); err != nil { + if panicFile, err := os.ReadFile(f); err != nil { l.Warnf("Support bundle: failed to load %s: %s", filepath.Base(f), err) } else { files = append(files, fileEntry{name: filepath.Base(f), data: panicFile}) @@ -1174,7 +1173,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) { } // Archived log (default on Windows) - if logFile, err := ioutil.ReadFile(locations.Get(locations.LogFile)); err == nil { + if logFile, err := os.ReadFile(locations.Get(locations.LogFile)); err == nil { files = append(files, fileEntry{name: "log-ondisk.txt", data: logFile}) } @@ -1233,7 +1232,7 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) { zipFilePath := filepath.Join(locations.GetBaseDir(locations.ConfigBaseDir), zipFileName) // Write buffer zip to local zip file (back up) - if err := ioutil.WriteFile(zipFilePath, zipFilesBuffer.Bytes(), 0600); err != nil { + if err := os.WriteFile(zipFilePath, zipFilesBuffer.Bytes(), 0600); err != nil { l.Warnln("Support bundle: support bundle zip could not be created:", err) } @@ -1323,7 +1322,7 @@ func (s *service) getDBIgnores(w http.ResponseWriter, r *http.Request) { func (s *service) postDBIgnores(w http.ResponseWriter, r *http.Request) { qs := r.URL.Query() - bs, err := ioutil.ReadAll(r.Body) + bs, err := io.ReadAll(r.Body) r.Body.Close() if err != nil { http.Error(w, err.Error(), 500) @@ -1614,7 +1613,7 @@ func (s *service) getFolderVersions(w http.ResponseWriter, r *http.Request) { func (s *service) postFolderVersionsRestore(w http.ResponseWriter, r *http.Request) { qs := r.URL.Query() - bs, err := ioutil.ReadAll(r.Body) + bs, err := io.ReadAll(r.Body) r.Body.Close() if err != nil { http.Error(w, err.Error(), 500) diff --git a/lib/api/api_test.go b/lib/api/api_test.go index f9ecc15c5cc..1bfe5df0839 100644 --- a/lib/api/api_test.go +++ b/lib/api/api_test.go @@ -13,7 +13,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net" "net/http" "net/http/httptest" @@ -223,7 +222,7 @@ func expectURLToContain(t *testing.T, url, exp string) { return } - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) res.Body.Close() if err != nil { t.Error(err) @@ -508,7 +507,7 @@ func testHTTPRequest(t *testing.T, baseURL string, tc httpTestCase, apikey strin return } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { t.Errorf("Unexpected error reading %s: %v", tc.URL, err) return @@ -1137,7 +1136,7 @@ func TestBrowse(t *testing.T) { pathSep := string(os.PathSeparator) - tmpDir, err := ioutil.TempDir("", "syncthing") + tmpDir, err := os.MkdirTemp("", "syncthing") if err != nil { t.Fatal(err) } @@ -1146,7 +1145,7 @@ func TestBrowse(t *testing.T) { if err := os.Mkdir(filepath.Join(tmpDir, "dir"), 0755); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(tmpDir, "file"), []byte("hello"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(tmpDir, "file"), []byte("hello"), 0644); err != nil { t.Fatal(err) } if err := os.Mkdir(filepath.Join(tmpDir, "MiXEDCase"), 0755); err != nil { @@ -1251,7 +1250,7 @@ func TestConfigChanges(t *testing.T) { APIKey: testAPIKey, }, } - tmpFile, err := ioutil.TempFile("", "syncthing-testConfig-") + tmpFile, err := os.CreateTemp("", "syncthing-testConfig-") if err != nil { panic(err) } @@ -1393,7 +1392,7 @@ func runningInContainer() bool { return false } - bs, err := ioutil.ReadFile("/proc/1/cgroup") + bs, err := os.ReadFile("/proc/1/cgroup") if err != nil { return false } diff --git a/lib/api/auto/auto_test.go b/lib/api/auto/auto_test.go index 2e997d5e3a3..db7e276f954 100644 --- a/lib/api/auto/auto_test.go +++ b/lib/api/auto/auto_test.go @@ -9,7 +9,7 @@ package auto_test import ( "bytes" "compress/gzip" - "io/ioutil" + "io" "strings" "testing" @@ -28,7 +28,7 @@ func TestAssets(t *testing.T) { var gr *gzip.Reader gr, _ = gzip.NewReader(strings.NewReader(idx.Content)) - html, _ := ioutil.ReadAll(gr) + html, _ := io.ReadAll(gr) if !bytes.Contains(html, []byte(" 0 { lr := io.LimitReader(f.rng, diff) - io.Copy(ioutil.Discard, lr) + io.Copy(io.Discard, lr) } f.offset = offs diff --git a/lib/fs/fakefs_test.go b/lib/fs/fakefs_test.go index 1b095f88128..63cae9d40fa 100644 --- a/lib/fs/fakefs_test.go +++ b/lib/fs/fakefs_test.go @@ -10,7 +10,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "path" "path/filepath" @@ -88,7 +87,7 @@ func TestFakeFS(t *testing.T) { } // Read - bs0, err := ioutil.ReadAll(fd) + bs0, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -101,7 +100,7 @@ func TestFakeFS(t *testing.T) { if err != nil { t.Fatal(err) } - bs1, err := ioutil.ReadAll(fd) + bs1, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -139,7 +138,7 @@ func testFakeFSRead(t *testing.T, fs Filesystem) { // Read fd.Seek(0, io.SeekStart) - bs0, err := ioutil.ReadAll(fd) + bs0, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -154,7 +153,7 @@ func testFakeFSRead(t *testing.T, fs Filesystem) { if n != len(buf0) { t.Fatal("short read") } - buf1, err := ioutil.ReadAll(fd) + buf1, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -252,7 +251,7 @@ func TestFakeFSCaseInsensitive(t *testing.T) { func createTestDir(t *testing.T) (string, bool) { t.Helper() - testDir, err := ioutil.TempDir("", "") + testDir, err := os.MkdirTemp("", "") if err != nil { t.Fatalf("could not create temporary dir for testing: %s", err) } @@ -328,7 +327,7 @@ func testFakeFSCaseInsensitive(t *testing.T, fs Filesystem) { t.Fatal(err) } - bs2, err := ioutil.ReadAll(fd2) + bs2, err := io.ReadAll(fd2) if err != nil { t.Fatal(err) } diff --git a/lib/fs/filesystem_copy_range_test.go b/lib/fs/filesystem_copy_range_test.go index e2db04fb928..5831753df7a 100644 --- a/lib/fs/filesystem_copy_range_test.go +++ b/lib/fs/filesystem_copy_range_test.go @@ -9,7 +9,6 @@ package fs import ( "bytes" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -257,7 +256,7 @@ func TestCopyRange(tttt *testing.T) { paths = []string{""} } for _, path := range paths { - testPath, err := ioutil.TempDir(path, "") + testPath, err := os.MkdirTemp(path, "") if err != nil { tttt.Fatal(err) } @@ -273,7 +272,7 @@ func TestCopyRange(tttt *testing.T) { tt.Run(testCase.name, func(t *testing.T) { srcBuf := make([]byte, testCase.srcSize) dstBuf := make([]byte, testCase.dstSize) - td, err := ioutil.TempDir(testPath, "") + td, err := os.MkdirTemp(testPath, "") if err != nil { t.Fatal(err) } diff --git a/lib/fs/mtimefs_test.go b/lib/fs/mtimefs_test.go index 3f7dc6c0704..bf14f514ef2 100644 --- a/lib/fs/mtimefs_test.go +++ b/lib/fs/mtimefs_test.go @@ -8,7 +8,6 @@ package fs import ( "errors" - "io/ioutil" "os" "path/filepath" "runtime" @@ -20,9 +19,9 @@ func TestMtimeFS(t *testing.T) { os.RemoveAll("testdata") defer os.RemoveAll("testdata") os.Mkdir("testdata", 0755) - ioutil.WriteFile("testdata/exists0", []byte("hello"), 0644) - ioutil.WriteFile("testdata/exists1", []byte("hello"), 0644) - ioutil.WriteFile("testdata/exists2", []byte("hello"), 0644) + os.WriteFile("testdata/exists0", []byte("hello"), 0644) + os.WriteFile("testdata/exists1", []byte("hello"), 0644) + os.WriteFile("testdata/exists2", []byte("hello"), 0644) // a random time with nanosecond precision testTime := time.Unix(1234567890, 123456789) @@ -83,7 +82,7 @@ func TestMtimeFS(t *testing.T) { } func TestMtimeFSWalk(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -93,7 +92,7 @@ func TestMtimeFSWalk(t *testing.T) { mtimefs := newMtimeFS(underlying, make(mapStore)) mtimefs.chtimes = failChtimes - if err := ioutil.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil { t.Fatal(err) } @@ -137,7 +136,7 @@ func TestMtimeFSWalk(t *testing.T) { } func TestMtimeFSOpen(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -147,7 +146,7 @@ func TestMtimeFSOpen(t *testing.T) { mtimefs := newMtimeFS(underlying, make(mapStore)) mtimefs.chtimes = failChtimes - if err := ioutil.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "file"), []byte("hello"), 0644); err != nil { t.Fatal(err) } @@ -200,7 +199,7 @@ func TestMtimeFSInsensitive(t *testing.T) { os.RemoveAll("testdata") defer os.RemoveAll("testdata") os.Mkdir("testdata", 0755) - ioutil.WriteFile("testdata/FiLe", []byte("hello"), 0644) + os.WriteFile("testdata/FiLe", []byte("hello"), 0644) // a random time with nanosecond precision testTime := time.Unix(1234567890, 123456789) diff --git a/lib/ignore/ignore_test.go b/lib/ignore/ignore_test.go index d9564ec6634..9781cbdfafc 100644 --- a/lib/ignore/ignore_test.go +++ b/lib/ignore/ignore_test.go @@ -10,7 +10,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -232,7 +231,7 @@ func TestCaseSensitivity(t *testing.T) { } func TestCaching(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -425,7 +424,7 @@ flamingo *.crow ` // Caches per file, hence write the patterns to a file. - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { b.Fatal(err) } @@ -466,7 +465,7 @@ flamingo } func TestCacheReload(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -989,7 +988,7 @@ func TestIssue4689(t *testing.T) { } func TestIssue4901(t *testing.T) { - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -1001,7 +1000,7 @@ func TestIssue4901(t *testing.T) { puppy ` - if err := ioutil.WriteFile(filepath.Join(dir, ".stignore"), []byte(stignore), 0777); err != nil { + if err := os.WriteFile(filepath.Join(dir, ".stignore"), []byte(stignore), 0777); err != nil { t.Fatalf(err.Error()) } @@ -1020,7 +1019,7 @@ func TestIssue4901(t *testing.T) { } } - if err := ioutil.WriteFile(filepath.Join(dir, "unicorn-lazor-death"), []byte(" "), 0777); err != nil { + if err := os.WriteFile(filepath.Join(dir, "unicorn-lazor-death"), []byte(" "), 0777); err != nil { t.Fatalf(err.Error()) } diff --git a/lib/logger/logger.go b/lib/logger/logger.go index cd3e3f23777..731be426b09 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -10,7 +10,6 @@ package logger import ( "fmt" "io" - "io/ioutil" "log" "os" "strings" @@ -75,7 +74,7 @@ func New() Logger { if os.Getenv("LOGGER_DISCARD") != "" { // Hack to completely disable logging, for example when running // benchmarks. - return newLogger(ioutil.Discard) + return newLogger(io.Discard) } return newLogger(controlStripper{os.Stdout}) } diff --git a/lib/logger/logger_test.go b/lib/logger/logger_test.go index 1937aa646db..cc3b72e67a0 100644 --- a/lib/logger/logger_test.go +++ b/lib/logger/logger_test.go @@ -6,7 +6,7 @@ package logger import ( "bytes" "fmt" - "io/ioutil" + "io" "log" "strings" "testing" @@ -186,12 +186,12 @@ func TestControlStripper(t *testing.T) { } func BenchmarkLog(b *testing.B) { - l := newLogger(controlStripper{ioutil.Discard}) + l := newLogger(controlStripper{io.Discard}) benchmarkLogger(b, l) } func BenchmarkLogNoStripper(b *testing.B) { - l := newLogger(ioutil.Discard) + l := newLogger(io.Discard) benchmarkLogger(b, l) } diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 06bf7d6d79d..c3a76424286 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -13,7 +13,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -680,8 +679,8 @@ func TestIssue3164(t *testing.T) { ignDir := filepath.Join("issue3164", "oktodelete") subDir := filepath.Join(ignDir, "foobar") must(t, ffs.MkdirAll(subDir, 0777)) - must(t, ioutil.WriteFile(filepath.Join(tmpDir, subDir, "file"), []byte("Hello"), 0644)) - must(t, ioutil.WriteFile(filepath.Join(tmpDir, ignDir, "file"), []byte("Hello"), 0644)) + must(t, os.WriteFile(filepath.Join(tmpDir, subDir, "file"), []byte("Hello"), 0644)) + must(t, os.WriteFile(filepath.Join(tmpDir, ignDir, "file"), []byte("Hello"), 0644)) file := protocol.FileInfo{ Name: "issue3164", } diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 121bcdffeec..78fa8e9f9af 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -12,7 +12,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -2152,7 +2151,7 @@ func TestIssue2782(t *testing.T) { if err := os.MkdirAll(testDir+"/syncdir", 0755); err != nil { t.Skip(err) } - if err := ioutil.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil { + if err := os.WriteFile(testDir+"/syncdir/file", []byte("hello, world\n"), 0644); err != nil { t.Skip(err) } if err := os.Symlink("syncdir", testDir+"/synclink"); err != nil { @@ -2763,7 +2762,7 @@ func TestVersionRestore(t *testing.T) { // In each file, we write the filename as the content // We verify that the content matches at the expected filenames // after the restore operation. - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") must(t, err) defer os.RemoveAll(dir) @@ -2900,7 +2899,7 @@ func TestVersionRestore(t *testing.T) { } defer fd.Close() - content, err := ioutil.ReadAll(fd) + content, err := io.ReadAll(fd) if err != nil { t.Error(err) } @@ -2930,7 +2929,7 @@ func TestVersionRestore(t *testing.T) { must(t, err) defer fd.Close() - content, err := ioutil.ReadAll(fd) + content, err := io.ReadAll(fd) if err != nil { t.Error(err) } diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index 35df4fdd9e9..76868748a2f 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -10,7 +10,6 @@ import ( "bytes" "context" "errors" - "io/ioutil" "os" "path/filepath" "runtime" @@ -236,7 +235,7 @@ func TestRequestVersioningSymlinkAttack(t *testing.T) { // Create a temporary directory that we will use as target to see if // we can escape to it - tmpdir, err := ioutil.TempDir("", "syncthing-test") + tmpdir, err := os.MkdirTemp("", "syncthing-test") if err != nil { t.Fatal(err) } @@ -681,7 +680,7 @@ func TestRequestSymlinkWindows(t *testing.T) { } func equalContents(path string, contents []byte) error { - if bs, err := ioutil.ReadFile(path); err != nil { + if bs, err := os.ReadFile(path); err != nil { return err } else if !bytes.Equal(bs, contents) { return errors.New("incorrect data") diff --git a/lib/model/testutils_test.go b/lib/model/testutils_test.go index 4ee390f5d8e..a31e57fdd32 100644 --- a/lib/model/testutils_test.go +++ b/lib/model/testutils_test.go @@ -8,7 +8,6 @@ package model import ( "context" - "io/ioutil" "os" "testing" "time" @@ -76,7 +75,7 @@ func init() { } func createTmpWrapper(cfg config.Configuration) (config.Wrapper, context.CancelFunc) { - tmpFile, err := ioutil.TempFile("", "syncthing-testConfig-") + tmpFile, err := os.CreateTemp("", "syncthing-testConfig-") if err != nil { panic(err) } @@ -215,7 +214,7 @@ func cleanupModelAndRemoveDir(m *testModel, dir string) { } func createTmpDir() string { - tmpDir, err := ioutil.TempDir("", "syncthing_testFolder-") + tmpDir, err := os.MkdirTemp("", "syncthing_testFolder-") if err != nil { panic("Failed to create temporary testing dir") } diff --git a/lib/nat/structs_test.go b/lib/nat/structs_test.go index 661a552f856..b43e52e08cf 100644 --- a/lib/nat/structs_test.go +++ b/lib/nat/structs_test.go @@ -7,7 +7,6 @@ package nat import ( - "io/ioutil" "net" "os" "testing" @@ -60,7 +59,7 @@ func TestMappingValidGateway(t *testing.T) { } func TestMappingClearAddresses(t *testing.T) { - tmpFile, err := ioutil.TempFile("", "syncthing-testConfig-") + tmpFile, err := os.CreateTemp("", "syncthing-testConfig-") if err != nil { t.Fatal(err) } diff --git a/lib/osutil/atomic.go b/lib/osutil/atomic.go index c2f6dd6e942..bf9730daf94 100644 --- a/lib/osutil/atomic.go +++ b/lib/osutil/atomic.go @@ -43,7 +43,7 @@ func CreateAtomic(path string) (*AtomicWriter, error) { // permissions. func CreateAtomicFilesystem(filesystem fs.Filesystem, path string) (*AtomicWriter, error) { // The security of this depends on the tempfile having secure - // permissions, 0600, from the beginning. This is what ioutil.TempFile + // permissions, 0600, from the beginning. This is what os.CreateTemp // does. We have a test that verifies that that is the case, should this // ever change in the standard library in the future. fd, err := TempFile(filesystem, filepath.Dir(path), TempPrefix) diff --git a/lib/osutil/atomic_test.go b/lib/osutil/atomic_test.go index 70c5b90d6ab..c9737784ad7 100644 --- a/lib/osutil/atomic_test.go +++ b/lib/osutil/atomic_test.go @@ -8,7 +8,6 @@ package osutil import ( "bytes" - "io/ioutil" "os" "path/filepath" "testing" @@ -35,7 +34,7 @@ func TestCreateAtomicCreate(t *testing.T) { t.Fatal("written bytes", n, "!= 5") } - if _, err := ioutil.ReadFile("testdata/file"); err == nil { + if _, err := os.ReadFile("testdata/file"); err == nil { t.Fatal("file should not exist") } @@ -43,7 +42,7 @@ func TestCreateAtomicCreate(t *testing.T) { t.Fatal(err) } - bs, err := ioutil.ReadFile("testdata/file") + bs, err := os.ReadFile("testdata/file") if err != nil { t.Fatal(err) } @@ -62,7 +61,7 @@ func TestCreateAtomicReplaceReadOnly(t *testing.T) { func testCreateAtomicReplace(t *testing.T, oldPerms os.FileMode) { t.Helper() - testdir, err := ioutil.TempDir("", "syncthing") + testdir, err := os.MkdirTemp("", "syncthing") if err != nil { t.Fatal(err) } @@ -75,7 +74,7 @@ func testCreateAtomicReplace(t *testing.T, oldPerms os.FileMode) { t.Fatal(err) } - if err := ioutil.WriteFile(testfile, []byte("some old data"), oldPerms); err != nil { + if err := os.WriteFile(testfile, []byte("some old data"), oldPerms); err != nil { t.Fatal(err) } @@ -103,7 +102,7 @@ func testCreateAtomicReplace(t *testing.T, oldPerms os.FileMode) { t.Fatal(err) } - bs, err := ioutil.ReadFile(testfile) + bs, err := os.ReadFile(testfile) if err != nil { t.Fatal(err) } diff --git a/lib/osutil/atomic_unix_test.go b/lib/osutil/atomic_unix_test.go index ec4f8e68e65..1d2e17da152 100644 --- a/lib/osutil/atomic_unix_test.go +++ b/lib/osutil/atomic_unix_test.go @@ -12,7 +12,6 @@ package osutil import ( - "io/ioutil" "os" "syscall" "testing" @@ -24,7 +23,7 @@ func TestTempFilePermissions(t *testing.T) { oldMask := syscall.Umask(0) defer syscall.Umask(oldMask) - fd, err := ioutil.TempFile("", "test") + fd, err := os.CreateTemp("", "test") if err != nil { t.Fatal(err) } diff --git a/lib/osutil/osutil_test.go b/lib/osutil/osutil_test.go index 772097e53a5..db1e608fddc 100644 --- a/lib/osutil/osutil_test.go +++ b/lib/osutil/osutil_test.go @@ -7,7 +7,7 @@ package osutil_test import ( - "io/ioutil" + "io" "os" "path/filepath" "runtime" @@ -83,7 +83,7 @@ func TestIsDeleted(t *testing.T) { func TestRenameOrCopy(t *testing.T) { mustTempDir := func() string { t.Helper() - tmpDir, err := ioutil.TempDir("", "") + tmpDir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func TestRenameOrCopy(t *testing.T) { if err != nil { t.Fatal(err) } - buf, err := ioutil.ReadAll(fd) + buf, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -147,7 +147,7 @@ func TestRenameOrCopy(t *testing.T) { if fd, err := test.dst.Open("new"); err != nil { t.Fatal(err) } else { - if buf, err := ioutil.ReadAll(fd); err != nil { + if buf, err := io.ReadAll(fd); err != nil { t.Fatal(err) } else if string(buf) != content { t.Fatalf("expected %s got %s", content, string(buf)) diff --git a/lib/osutil/traversessymlink_test.go b/lib/osutil/traversessymlink_test.go index 5bf6e7b59f4..374c471fd58 100644 --- a/lib/osutil/traversessymlink_test.go +++ b/lib/osutil/traversessymlink_test.go @@ -7,7 +7,6 @@ package osutil_test import ( - "io/ioutil" "os" "path/filepath" "runtime" @@ -18,7 +17,7 @@ import ( ) func TestTraversesSymlink(t *testing.T) { - tmpDir, err := ioutil.TempDir(".", ".test-TraversesSymlink-") + tmpDir, err := os.MkdirTemp(".", ".test-TraversesSymlink-") if err != nil { panic("Failed to create temporary testing dir") } @@ -71,7 +70,7 @@ func TestTraversesSymlink(t *testing.T) { } func TestIssue4875(t *testing.T) { - tmpDir, err := ioutil.TempDir("", ".test-Issue4875-") + tmpDir, err := os.MkdirTemp("", ".test-Issue4875-") if err != nil { panic("Failed to create temporary testing dir") } diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index 1c1d617f75f..81a74bb9f9a 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -10,7 +10,7 @@ import ( "encoding/json" "errors" "io" - "io/ioutil" + "os" "runtime" "sync" "testing" @@ -432,8 +432,8 @@ func testMarshal(t *testing.T, prefix string, m1, m2 message) bool { bs1, _ := json.MarshalIndent(m1, "", " ") bs2, _ := json.MarshalIndent(m2, "", " ") if !bytes.Equal(bs1, bs2) { - ioutil.WriteFile(prefix+"-1.txt", bs1, 0644) - ioutil.WriteFile(prefix+"-2.txt", bs2, 0644) + os.WriteFile(prefix+"-1.txt", bs1, 0644) + os.WriteFile(prefix+"-2.txt", bs2, 0644) return false } diff --git a/lib/rc/rc.go b/lib/rc/rc.go index f21d0e31df7..875ee849e7c 100644 --- a/lib/rc/rc.go +++ b/lib/rc/rc.go @@ -14,7 +14,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net/http" "net/url" @@ -393,7 +392,7 @@ func (p *Process) Model(folder string) (Model, error) { } func (p *Process) readResponse(resp *http.Response) ([]byte, error) { - bs, err := ioutil.ReadAll(resp.Body) + bs, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { return bs, err diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index ba7e9a33138..4edad2b4778 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -13,7 +13,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -376,7 +375,7 @@ func TestWalkSymlinkWindows(t *testing.T) { func TestWalkRootSymlink(t *testing.T) { // Create a folder with a symlink in it - tmp, err := ioutil.TempDir("", "") + tmp, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -709,7 +708,7 @@ func TestStopWalk(t *testing.T) { } func TestIssue4799(t *testing.T) { - tmp, err := ioutil.TempDir("", "") + tmp, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -771,7 +770,7 @@ func TestRecurseInclude(t *testing.T) { } func TestIssue4841(t *testing.T) { - tmp, err := ioutil.TempDir("", "") + tmp, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } diff --git a/lib/syncthing/syncthing_test.go b/lib/syncthing/syncthing_test.go index 8add69ee821..e9feedf4487 100644 --- a/lib/syncthing/syncthing_test.go +++ b/lib/syncthing/syncthing_test.go @@ -7,7 +7,6 @@ package syncthing import ( - "io/ioutil" "os" "testing" "time" @@ -22,7 +21,7 @@ import ( func tempCfgFilename(t *testing.T) string { t.Helper() - f, err := ioutil.TempFile("", "syncthing-testConfig-") + f, err := os.CreateTemp("", "syncthing-testConfig-") if err != nil { t.Fatal(err) } diff --git a/lib/syncthing/utils.go b/lib/syncthing/utils.go index eecb563942f..5f59af2b64c 100644 --- a/lib/syncthing/utils.go +++ b/lib/syncthing/utils.go @@ -10,7 +10,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "os" "github.com/pkg/errors" @@ -133,12 +132,12 @@ func archiveAndSaveConfig(cfg config.Wrapper, originalVersion int) error { } func copyFile(src, dst string) error { - bs, err := ioutil.ReadFile(src) + bs, err := os.ReadFile(src) if err != nil { return err } - if err := ioutil.WriteFile(dst, bs, 0600); err != nil { + if err := os.WriteFile(dst, bs, 0600); err != nil { // Attempt to clean up os.Remove(dst) return err diff --git a/lib/upgrade/upgrade_supported.go b/lib/upgrade/upgrade_supported.go index 45eeaf426ce..80312a6facd 100644 --- a/lib/upgrade/upgrade_supported.go +++ b/lib/upgrade/upgrade_supported.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "path" @@ -284,7 +283,7 @@ func readTarGz(archiveName, dir string, r io.Reader) (string, error) { } func readZip(archiveName, dir string, r io.Reader) (string, error) { - body, err := ioutil.ReadAll(r) + body, err := io.ReadAll(r) if err != nil { return "", err } @@ -357,7 +356,7 @@ func archiveFileVisitor(dir string, tempFile *string, signature *[]byte, archive case "release.sig": l.Debugf("found signature %s", archivePath) - *signature, err = ioutil.ReadAll(io.LimitReader(filedata, maxSignatureSize)) + *signature, err = io.ReadAll(io.LimitReader(filedata, maxSignatureSize)) if err != nil { return err } @@ -407,7 +406,7 @@ func verifyUpgrade(archiveName, tempName string, sig []byte) error { func writeBinary(dir string, inFile io.Reader) (filename string, err error) { // Write the binary to a temporary file. - outFile, err := ioutil.TempFile(dir, "syncthing") + outFile, err := os.CreateTemp(dir, "syncthing") if err != nil { return "", err } diff --git a/lib/upnp/upnp.go b/lib/upnp/upnp.go index 72fc22fedc9..9cf3b5afb42 100644 --- a/lib/upnp/upnp.go +++ b/lib/upnp/upnp.go @@ -38,7 +38,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "io" "net" "net/http" "net/url" @@ -467,7 +467,7 @@ func soapRequest(ctx context.Context, url, service, function, message string) ([ return resp, err } - resp, _ = ioutil.ReadAll(r.Body) + resp, _ = io.ReadAll(r.Body) l.Debugf("SOAP Response: %s\n\n%s\n\n", r.Status, resp) r.Body.Close() diff --git a/lib/versioner/external_test.go b/lib/versioner/external_test.go index 5979d5e7659..e4283fa3e97 100644 --- a/lib/versioner/external_test.go +++ b/lib/versioner/external_test.go @@ -7,7 +7,6 @@ package versioner import ( - "io/ioutil" "os" "path/filepath" "runtime" @@ -85,7 +84,7 @@ func prepForRemoval(t *testing.T, file string) { if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(file, []byte("hello\n"), 0644); err != nil { + if err := os.WriteFile(file, []byte("hello\n"), 0644); err != nil { t.Fatal(err) } } diff --git a/lib/versioner/simple_test.go b/lib/versioner/simple_test.go index 21949213806..12dc2c81ac2 100644 --- a/lib/versioner/simple_test.go +++ b/lib/versioner/simple_test.go @@ -7,8 +7,8 @@ package versioner import ( - "io/ioutil" "math" + "os" "path/filepath" "testing" "time" @@ -55,7 +55,7 @@ func TestSimpleVersioningVersionCount(t *testing.T) { t.Skip("Test takes some time, skipping.") } - dir, err := ioutil.TempDir("", "") + dir, err := os.MkdirTemp("", "") //defer os.RemoveAll(dir) if err != nil { t.Error(err) diff --git a/lib/versioner/staggered_test.go b/lib/versioner/staggered_test.go index fbf78d4dfe0..360c29ceb06 100644 --- a/lib/versioner/staggered_test.go +++ b/lib/versioner/staggered_test.go @@ -7,7 +7,7 @@ package versioner import ( - "io/ioutil" + "os" "path/filepath" "sort" "strconv" @@ -133,11 +133,11 @@ func TestCreateVersionPath(t *testing.T) { ) // Create a test dir and file - tmpDir, err := ioutil.TempDir("", "") + tmpDir, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(tmpDir, archiveFile), []byte("sup"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(tmpDir, archiveFile), []byte("sup"), 0644); err != nil { t.Fatal(err) } diff --git a/lib/versioner/trashcan_test.go b/lib/versioner/trashcan_test.go index eb59f0954ad..891c64ecf3f 100644 --- a/lib/versioner/trashcan_test.go +++ b/lib/versioner/trashcan_test.go @@ -7,7 +7,8 @@ package versioner import ( - "io/ioutil" + "io" + "os" "testing" "time" @@ -19,12 +20,12 @@ func TestTrashcanArchiveRestoreSwitcharoo(t *testing.T) { // This tests that trashcan versioner restoration correctly archives existing file, because trashcan versioner // files are untagged, archiving existing file to replace with a restored version technically should collide in // in names. - tmpDir1, err := ioutil.TempDir("", "") + tmpDir1, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } - tmpDir2, err := ioutil.TempDir("", "") + tmpDir2, err := os.MkdirTemp("", "") if err != nil { t.Fatal(err) } @@ -105,7 +106,7 @@ func readFile(t *testing.T, filesystem fs.Filesystem, name string) string { t.Fatal(err) } defer fd.Close() - buf, err := ioutil.ReadAll(fd) + buf, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } diff --git a/lib/versioner/versioner_test.go b/lib/versioner/versioner_test.go index 4dfaaf32225..4ba73c36c6a 100644 --- a/lib/versioner/versioner_test.go +++ b/lib/versioner/versioner_test.go @@ -9,7 +9,6 @@ package versioner import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -56,7 +55,7 @@ func TestVersionerCleanOut(t *testing.T) { oldTime := time.Now().Add(-8 * 24 * time.Hour) for file, shouldRemove := range testcases { os.MkdirAll(filepath.Dir(file), 0777) - if err := ioutil.WriteFile(file, []byte("data"), 0644); err != nil { + if err := os.WriteFile(file, []byte("data"), 0644); err != nil { t.Fatal(err) } if shouldRemove { diff --git a/lib/weakhash/weakhash_test.go b/lib/weakhash/weakhash_test.go index 0d9ca91c295..78c2d3906dc 100644 --- a/lib/weakhash/weakhash_test.go +++ b/lib/weakhash/weakhash_test.go @@ -13,7 +13,6 @@ import ( "bytes" "context" "io" - "io/ioutil" "os" "reflect" "testing" @@ -22,7 +21,7 @@ import ( var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz") func TestFinder(t *testing.T) { - f, err := ioutil.TempFile("", "") + f, err := os.CreateTemp("", "") if err != nil { t.Error(err) } diff --git a/script/authors.go b/script/authors.go index 3068aebd56b..37fa76bbedc 100644 --- a/script/authors.go +++ b/script/authors.go @@ -15,7 +15,7 @@ package main import ( "bytes" "fmt" - "io/ioutil" + "io" "log" "math" "os" @@ -108,7 +108,7 @@ func main() { bs := readAll(htmlFile) bs = authorsRe.ReplaceAll(bs, []byte("id=\"contributor-list\">\n"+replacement+"\n ")) - if err := ioutil.WriteFile(htmlFile, bs, 0644); err != nil { + if err := os.WriteFile(htmlFile, bs, 0644); err != nil { log.Fatal(err) } @@ -173,7 +173,7 @@ func readAll(path string) []byte { } defer fd.Close() - bs, err := ioutil.ReadAll(fd) + bs, err := io.ReadAll(fd) if err != nil { log.Fatal(err) } diff --git a/script/commit-msg.go b/script/commit-msg.go index c5e178dea42..fa1a13d167f 100644 --- a/script/commit-msg.go +++ b/script/commit-msg.go @@ -12,7 +12,6 @@ package main import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -31,7 +30,7 @@ func main() { os.Exit(exitError) } - bs, err := ioutil.ReadFile(os.Args[1]) + bs, err := os.ReadFile(os.Args[1]) if err != nil { fmt.Println("Reading input:", err) os.Exit(exitError) diff --git a/script/genassets.go b/script/genassets.go index 71f965ab941..4d1b636f62d 100644 --- a/script/genassets.go +++ b/script/genassets.go @@ -16,7 +16,6 @@ import ( "fmt" "go/format" "io" - "io/ioutil" "os" "path/filepath" "strconv" @@ -74,7 +73,7 @@ func walkerFor(basePath string) filepath.WalkFunc { } if info.Mode().IsRegular() { - data, err := ioutil.ReadFile(name) + data, err := os.ReadFile(name) if err != nil { return err } diff --git a/script/prune_mocks.go b/script/prune_mocks.go index ba924102778..ab56c3e08a9 100644 --- a/script/prune_mocks.go +++ b/script/prune_mocks.go @@ -12,7 +12,6 @@ package main import ( "bufio" "flag" - "io/ioutil" "log" "os" "os/exec" @@ -51,7 +50,7 @@ func pruneInterfaceCheck(path string, size int64) error { } defer fd.Close() - tmp, err := ioutil.TempFile(".", "") + tmp, err := os.CreateTemp(".", "") if err != nil { return err } diff --git a/script/transifexdl.go b/script/transifexdl.go index 321bdab6fc8..abb5291cfc1 100644 --- a/script/transifexdl.go +++ b/script/transifexdl.go @@ -12,7 +12,7 @@ package main import ( "encoding/json" "fmt" - "io/ioutil" + "io" "log" "net/http" "os" @@ -140,7 +140,7 @@ func loadValidLangs() []string { log.Fatal(err) } defer fd.Close() - bs, err := ioutil.ReadAll(fd) + bs, err := io.ReadAll(fd) if err != nil { log.Fatal(err) } diff --git a/test/conflict_test.go b/test/conflict_test.go index c163dbb561f..de93fe3542a 100644 --- a/test/conflict_test.go +++ b/test/conflict_test.go @@ -11,7 +11,6 @@ package integration import ( "bytes" - "io/ioutil" "log" "os" "path/filepath" @@ -189,7 +188,7 @@ func TestConflictsDefault(t *testing.T) { if len(files) != 1 { t.Errorf("Expected 1 conflicted files instead of %d", len(files)) } - bs, err := ioutil.ReadFile("s1/testfile.txt") + bs, err := os.ReadFile("s1/testfile.txt") if err != nil { t.Error("reading file:", err) } @@ -216,26 +215,26 @@ func TestConflictsInitialMerge(t *testing.T) { // File 1 is a conflict - err = ioutil.WriteFile("s1/file1", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file1", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile("s2/file1", []byte("goodbye\n"), 0644) + err = os.WriteFile("s2/file1", []byte("goodbye\n"), 0644) if err != nil { t.Fatal(err) } // File 2 exists on s1 only - err = ioutil.WriteFile("s1/file2", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file2", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } // File 3 exists on s2 only - err = ioutil.WriteFile("s2/file3", []byte("goodbye\n"), 0644) + err = os.WriteFile("s2/file3", []byte("goodbye\n"), 0644) if err != nil { t.Fatal(err) } @@ -315,15 +314,15 @@ func TestConflictsIndexReset(t *testing.T) { // Three files on s1 - err = ioutil.WriteFile("s1/file1", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file1", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile("s1/file2", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file2", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile("s2/file3", []byte("hello\n"), 0644) + err = os.WriteFile("s2/file3", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } @@ -371,7 +370,7 @@ func TestConflictsIndexReset(t *testing.T) { // locally after we rest the index, unless we have a fix for that. for i := 0; i < 5; i++ { - err = ioutil.WriteFile("s2/file2", []byte("hello1\n"), 0644) + err = os.WriteFile("s2/file2", []byte("hello1\n"), 0644) if err != nil { t.Fatal(err) } @@ -393,7 +392,7 @@ func TestConflictsIndexReset(t *testing.T) { // s1/file1 (remote) changes while receiver is down - err = ioutil.WriteFile("s1/file1", []byte("goodbye\n"), 0644) + err = os.WriteFile("s1/file1", []byte("goodbye\n"), 0644) if err != nil { t.Fatal(err) } @@ -406,7 +405,7 @@ func TestConflictsIndexReset(t *testing.T) { // s2/file2 (local) changes while receiver is down - err = ioutil.WriteFile("s2/file2", []byte("goodbye\n"), 0644) + err = os.WriteFile("s2/file2", []byte("goodbye\n"), 0644) if err != nil { t.Fatal(err) } @@ -468,22 +467,22 @@ func TestConflictsSameContent(t *testing.T) { // Two files on s1 - err = ioutil.WriteFile("s1/file1", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file1", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile("s1/file2", []byte("hello\n"), 0644) + err = os.WriteFile("s1/file2", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } // Two files on s2, content differs in file1 only, timestamps differ on both. - err = ioutil.WriteFile("s2/file1", []byte("goodbye\n"), 0644) + err = os.WriteFile("s2/file1", []byte("goodbye\n"), 0644) if err != nil { t.Fatal(err) } - err = ioutil.WriteFile("s2/file2", []byte("hello\n"), 0644) + err = os.WriteFile("s2/file2", []byte("hello\n"), 0644) if err != nil { t.Fatal(err) } diff --git a/test/delay_scan_test.go b/test/delay_scan_test.go index 8b451c9824e..1cb90e2c179 100644 --- a/test/delay_scan_test.go +++ b/test/delay_scan_test.go @@ -10,8 +10,8 @@ package integration import ( - "io/ioutil" "log" + "os" "sync" "testing" "time" @@ -31,7 +31,7 @@ func TestRescanWithDelay(t *testing.T) { } log.Println("Generating .stignore...") - err = ioutil.WriteFile("s1/.stignore", []byte("some ignore data\n"), 0644) + err = os.WriteFile("s1/.stignore", []byte("some ignore data\n"), 0644) if err != nil { t.Fatal(err) } diff --git a/test/http_test.go b/test/http_test.go index a00bdb7d2ef..4541c75b472 100644 --- a/test/http_test.go +++ b/test/http_test.go @@ -11,8 +11,9 @@ package integration import ( "bytes" - "io/ioutil" + "io" "net/http" + "os" "strings" "testing" @@ -33,7 +34,7 @@ func TestHTTPGetIndex(t *testing.T) { if res.StatusCode != 200 { t.Errorf("Status %d != 200", res.StatusCode) } - bs, err := ioutil.ReadAll(res.Body) + bs, err := io.ReadAll(res.Body) if err != nil { t.Fatal(err) } @@ -57,7 +58,7 @@ func TestHTTPGetIndex(t *testing.T) { if res.StatusCode != 200 { t.Errorf("Status %d != 200", res.StatusCode) } - bs, err = ioutil.ReadAll(res.Body) + bs, err = io.ReadAll(res.Body) if err != nil { t.Fatal(err) } @@ -222,7 +223,7 @@ func setupAPIBench() *rc.Process { panic(err) } - err = ioutil.WriteFile("s1/knownfile", []byte("somedatahere"), 0644) + err = os.WriteFile("s1/knownfile", []byte("somedatahere"), 0644) if err != nil { panic(err) } diff --git a/test/ignore_test.go b/test/ignore_test.go index 5022026c0be..abc2faab419 100644 --- a/test/ignore_test.go +++ b/test/ignore_test.go @@ -10,7 +10,6 @@ package integration import ( - "io/ioutil" "log" "os" "path/filepath" @@ -67,7 +66,7 @@ func TestIgnores(t *testing.T) { // Add some of them to an ignore file - err = ioutil.WriteFile("s1/.stignore", + err = os.WriteFile("s1/.stignore", []byte("f1*\nf2\nd1*\nd2\ns1*\ns2\n(?i)*.txt"), // [fds][34] only non-ignored items 0644) if err != nil { @@ -92,7 +91,7 @@ func TestIgnores(t *testing.T) { // Change the pattern to include some of the files and dirs previously ignored time.Sleep(1100 * time.Millisecond) - err = ioutil.WriteFile("s1/.stignore", []byte("f2\nd2\ns2\n"), 0644) + err = os.WriteFile("s1/.stignore", []byte("f2\nd2\ns2\n"), 0644) // Rescan and verify that we see them diff --git a/test/override_test.go b/test/override_test.go index b2fa14f11cd..5851dfbe0db 100644 --- a/test/override_test.go +++ b/test/override_test.go @@ -10,7 +10,7 @@ package integration import ( - "io/ioutil" + "io" "log" "os" "strings" @@ -127,7 +127,7 @@ func TestOverride(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err := ioutil.ReadAll(fd) + bs, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -141,7 +141,7 @@ func TestOverride(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err = ioutil.ReadAll(fd) + bs, err = io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -295,7 +295,7 @@ func TestOverrideIgnores(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err := ioutil.ReadAll(fd) + bs, err := io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -309,7 +309,7 @@ func TestOverrideIgnores(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err = ioutil.ReadAll(fd) + bs, err = io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -337,7 +337,7 @@ func TestOverrideIgnores(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err = ioutil.ReadAll(fd) + bs, err = io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -388,7 +388,7 @@ func TestOverrideIgnores(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err = ioutil.ReadAll(fd) + bs, err = io.ReadAll(fd) if err != nil { t.Fatal(err) } @@ -423,7 +423,7 @@ func TestOverrideIgnores(t *testing.T) { if err != nil { t.Fatal(err) } - bs, err = ioutil.ReadAll(fd) + bs, err = io.ReadAll(fd) if err != nil { t.Fatal(err) } diff --git a/test/parallell_scan_test.go b/test/parallell_scan_test.go index e3f35f2b710..13a07d5b639 100644 --- a/test/parallell_scan_test.go +++ b/test/parallell_scan_test.go @@ -10,8 +10,8 @@ package integration import ( - "io/ioutil" "log" + "os" "sync" "testing" "time" @@ -31,7 +31,7 @@ func TestRescanInParallel(t *testing.T) { } log.Println("Generating .stignore...") - err = ioutil.WriteFile("s1/.stignore", []byte("some ignore data\n"), 0644) + err = os.WriteFile("s1/.stignore", []byte("some ignore data\n"), 0644) if err != nil { t.Fatal(err) } diff --git a/test/reset_test.go b/test/reset_test.go index 1e781c7e385..35d471571ef 100644 --- a/test/reset_test.go +++ b/test/reset_test.go @@ -13,7 +13,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -147,7 +146,7 @@ func createFiles(t *testing.T) int { const n = 8 for i := 0; i < n; i++ { file := fmt.Sprintf("f%d", i) - if err := ioutil.WriteFile(filepath.Join("s1", file), []byte("data"), 0644); err != nil { + if err := os.WriteFile(filepath.Join("s1", file), []byte("data"), 0644); err != nil { t.Fatal(err) } } diff --git a/test/util.go b/test/util.go index 8f69dacdaa1..99a0f73861a 100644 --- a/test/util.go +++ b/test/util.go @@ -14,7 +14,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "math/rand" "os" @@ -206,7 +205,7 @@ func alterFiles(dir string) error { return err } d1 := []byte("I used to be a dir: " + path) - err := ioutil.WriteFile(path, d1, 0644) + err := os.WriteFile(path, d1, 0644) if err != nil { return err } @@ -551,7 +550,7 @@ func startInstance(t *testing.T, i int) *rc.Process { } func symlinksSupported() bool { - tmp, err := ioutil.TempDir("", "symlink-test") + tmp, err := os.MkdirTemp("", "symlink-test") if err != nil { return false } From 1754c933706792c3945009d87d1e560cead1ade1 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 22 Nov 2021 09:38:24 +0100 Subject: [PATCH 017/220] lib/config, lib/ignore: Write Windows line endings (fixes #7115) (#8052) --- build.go | 2 +- lib/config/config_test.go | 35 +++++++++++++++++++++++++++++++ lib/config/wrapper.go | 2 +- lib/ignore/ignore.go | 3 ++- lib/ignore/ignore_test.go | 39 +++++++++++++++++++++++++++++++++++ lib/osutil/replacingwriter.go | 14 +++++++++++++ 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/build.go b/build.go index 2a480a5464d..c2e10593982 100644 --- a/build.go +++ b/build.go @@ -1293,7 +1293,7 @@ func zipFile(out string, files []archiveFile) { if err != nil { log.Fatal(err) } - bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1) + bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\r', '\n'}, -1) fh.UncompressedSize = uint32(len(bs)) fh.UncompressedSize64 = uint64(len(bs)) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index ec7db576304..861c5783980 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -594,6 +594,41 @@ func TestNewSaveLoad(t *testing.T) { } } +func TestWindowsLineEndings(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Windows specific") + } + + dir, err := os.MkdirTemp("", "syncthing-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + path := filepath.Join(dir, "config.xml") + os.Remove(path) + defer os.Remove(path) + + intCfg := New(device1) + cfg := wrap(path, intCfg, device1) + defer cfg.stop() + + if err := cfg.Save(); err != nil { + t.Error(err) + } + + bs, err := os.ReadFile(path) + if err != nil { + t.Error(err) + } + + unixLineEndings := bytes.Count(bs, []byte("\n")) + windowsLineEndings := bytes.Count(bs, []byte("\r\n")) + if unixLineEndings == 0 || windowsLineEndings != unixLineEndings { + t.Error("expected there to be a non-zero number of Windows line endings") + } +} + func TestPrepare(t *testing.T) { var cfg Configuration diff --git a/lib/config/wrapper.go b/lib/config/wrapper.go index 4d5c8f93ccf..4c52b0ce069 100644 --- a/lib/config/wrapper.go +++ b/lib/config/wrapper.go @@ -502,7 +502,7 @@ func (w *wrapper) Save() error { return err } - if err := w.cfg.WriteXML(fd); err != nil { + if err := w.cfg.WriteXML(osutil.LineEndingsWriter(fd)); err != nil { l.Debugln("WriteXML:", err) fd.Close() return err diff --git a/lib/ignore/ignore.go b/lib/ignore/ignore.go index 9b13c1e6315..243f178d925 100644 --- a/lib/ignore/ignore.go +++ b/lib/ignore/ignore.go @@ -595,8 +595,9 @@ func WriteIgnores(filesystem fs.Filesystem, path string, content []string) error return err } + wr := osutil.LineEndingsWriter(fd) for _, line := range content { - fmt.Fprintln(fd, line) + fmt.Fprintln(wr, line) } if err := fd.Close(); err != nil { diff --git a/lib/ignore/ignore_test.go b/lib/ignore/ignore_test.go index 9781cbdfafc..b1f19c5a99d 100644 --- a/lib/ignore/ignore_test.go +++ b/lib/ignore/ignore_test.go @@ -1192,3 +1192,42 @@ func TestEmptyPatterns(t *testing.T) { } } } + +func TestWindowsLineEndings(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Windows specific") + } + + lines := "foo\nbar\nbaz\n" + + dir, err := os.MkdirTemp("", "syncthing-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + ffs := fs.NewFilesystem(fs.FilesystemTypeBasic, dir) + m := New(ffs) + if err := m.Parse(strings.NewReader(lines), ".stignore"); err != nil { + t.Fatal(err) + } + if err := WriteIgnores(ffs, ".stignore", m.Lines()); err != nil { + t.Fatal(err) + } + + fd, err := ffs.Open(".stignore") + if err != nil { + t.Fatal(err) + } + bs, err := io.ReadAll(fd) + fd.Close() + if err != nil { + t.Fatal(err) + } + + unixLineEndings := bytes.Count(bs, []byte("\n")) + windowsLineEndings := bytes.Count(bs, []byte("\r\n")) + if unixLineEndings == 0 || windowsLineEndings != unixLineEndings { + t.Error("expected there to be a non-zero number of Windows line endings") + } +} diff --git a/lib/osutil/replacingwriter.go b/lib/osutil/replacingwriter.go index 71e86fd631d..18be86f3eb1 100644 --- a/lib/osutil/replacingwriter.go +++ b/lib/osutil/replacingwriter.go @@ -9,6 +9,7 @@ package osutil import ( "bytes" "io" + "runtime" ) type ReplacingWriter struct { @@ -46,3 +47,16 @@ func (w ReplacingWriter) Write(bs []byte) (int, error) { return written, err } + +// LineEndingsWriter returns a writer that writes platform-appropriate line +// endings. (This is a no-op on non-Windows platforms.) +func LineEndingsWriter(w io.Writer) io.Writer { + if runtime.GOOS != "windows" { + return w + } + return &ReplacingWriter{ + Writer: w, + From: '\n', + To: []byte{'\r', '\n'}, + } +} From 7161a99b04b1370aeed3e7fb39c8d7f370e4307b Mon Sep 17 00:00:00 2001 From: tomasz1986 Date: Mon, 22 Nov 2021 17:41:43 +0900 Subject: [PATCH 018/220] gui: Allow text wrapping in remove folder/device modals (#7976) Remove the embedded CSS that prevents the lines from wrapping, resulting in device and folder names being hidden on small screens. --- gui/default/syncthing/device/removeDeviceDialogView.html | 8 +++++--- gui/default/syncthing/folder/removeFolderDialogView.html | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/gui/default/syncthing/device/removeDeviceDialogView.html b/gui/default/syncthing/device/removeDeviceDialogView.html index bc9f20292c5..c8197d3ce24 100644 --- a/gui/default/syncthing/device/removeDeviceDialogView.html +++ b/gui/default/syncthing/device/removeDeviceDialogView.html @@ -1,9 +1,11 @@ -
+
@@ -172,7 +172,7 @@ -
+
diff --git a/gui/default/syncthing/folder/editFolderModalView.html b/gui/default/syncthing/folder/editFolderModalView.html index 5fd8bc8e382..2c0c3559750 100644 --- a/gui/default/syncthing/folder/editFolderModalView.html +++ b/gui/default/syncthing/folder/editFolderModalView.html @@ -2,44 +2,44 @@