Merge pull request #2195 from ifedorenko/out-of-order-restore-no-prog…

…ress

restorer: allow writing target file blobs out of order

changelog/unreleased/pull-2195

@@ -0,0 +1,17 @@
+Enhancement: Simplify and improve restore performance
+
+Significantly improves restore performance of large files (i.e. 50M+):
+https://github.com/restic/restic/issues/2074
+https://forum.restic.net/t/restore-using-rclone-gdrive-backend-is-slow/1112/8
+https://forum.restic.net/t/degraded-restore-performance-s3-backend/1400
+
+Fixes "not enough cache capacity" error during restore:
+https://github.com/restic/restic/issues/2244
+
+NOTE: This new implementation does not guarantee order in which blobs
+are written to the target files and, for example, the last blob of a
+file can be written to the file before any of the preceeding file blobs.
+It is therefore possible to have gaps in the data written to the target
+files if restore fails or interrupted by the user.
+
+https://github.com/restic/restic/pull/2195

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Azure/azure-sdk-for-go v27.3.0+incompatible
 	github.com/Azure/go-autorest/autorest v0.9.2 // indirect
 	github.com/cenkalti/backoff v2.1.1+incompatible
+	github.com/cespare/xxhash v1.1.0
 	github.com/cpuguy83/go-md2man v1.0.10 // indirect
 	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/elithrar/simple-scrypt v1.3.0

go.sum

@@ -20,6 +20,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -28,6 +30,8 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
 github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
@@ -145,6 +149,8 @@ github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHah
 github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=

internal/restorer/doc.go

@@ -5,29 +5,20 @@
 // request and avoiding repeated downloads of the same pack. In addition,
 // several pack files are fetched concurrently.
 //
-// Here is high-level pseudo-code of the how the Restorer attempts to achieve
+// Here is high-level pseudo-code of how the Restorer attempts to achieve
 // these goals:
 //
 //   while there are packs to process
 //     choose a pack to process                      [1]
-//     get the pack from the backend or cache        [2]
+//     retrieve the pack from the backend            [2]
 //     write pack blobs to the files that need them  [3]
-//     if not all pack blobs were used
-//       cache the pack for future use               [4]
 //
-// Pack download and processing (steps [2] - [4]) runs on multiple concurrent
-// Goroutines. The Restorer runs all steps [2]-[4] sequentially on the same
-// Goroutine.
+// Retrieval of repository packs (step [2]) and writing target files (step [3])
+// are performed concurrently on multiple goroutines.
 //
-// Before a pack is downloaded (step [2]), the required space is "reserved" in
-// the pack cache. Actual download uses single backend request to get all
-// required pack blobs. This may download blobs that are not needed, but we
-// assume it'll still be faster than getting individual blobs.
-//
-// Target files are written (step [3]) in the "right" order, first file blob
-// first, then second, then third and so on. Blob write order implies that some
-// pack blobs may not be immediately used, i.e. they are "out of order" for
-// their respective target files. Packs with unused blobs are cached (step
-// [4]). The cache has capacity limit and may purge packs before they are fully
-// used, in which case the purged packs will need to be re-downloaded.
+// Implementation does not guarantee order in which blobs are written to the
+// target files and, for example, the last blob of a file can be written to the
+// file before any of the preceeding file blobs. It is therefore possible to
+// have gaps in the data written to the target files if restore fails or
+// interrupted by the user.
 package restorer