Added support for Google Cloud Storage repositories using the existin…

…g s3 support

to address restic#211.

Added a new package to handle GCS configuration. GCS repositories get specified
using gs://bucketname/prefix syntax, similar to gsutil.

Added tests for the various cases to config_test and location_test.

src/cmds/restic/global.go

@@ -329,6 +329,17 @@ func open(s string) (restic.Backend, error) {
 
 		debug.Log("opening s3 repository at %#v", cfg)
 		be, err = s3.Open(cfg)
+	case "gs":
+		cfg := loc.Config.(s3.Config)
+		if cfg.KeyID == "" {
+			cfg.KeyID = os.Getenv("GS_ACCESS_KEY_ID")
+
+		}
+		if cfg.Secret == "" {
+			cfg.Secret = os.Getenv("GS_SECRET_ACCESS_KEY")
+		}
+		debug.Log("open", "opening gcs repository at %#v", cfg)
+		be, err := s3.Open(cfg)
 	case "rest":
 		be, err = rest.Open(loc.Config.(rest.Config))
 	default:
@@ -369,6 +380,18 @@ func create(s string) (restic.Backend, error) {
 
 		debug.Log("create s3 repository at %#v", loc.Config)
 		return s3.Open(cfg)
+	case "gs":
+		cfg := loc.Config.(s3.Config)
+		if cfg.KeyID == "" {
+			cfg.KeyID = os.Getenv("GS_ACCESS_KEY_ID")
+
+		}
+		if cfg.Secret == "" {
+			cfg.Secret = os.Getenv("GS_SECRET_ACCESS_KEY")
+		}
+
+		debug.Log("open", "create gcs repository at %#v", loc.Config)
+		return s3.Open(cfg)
 	case "rest":
 		return rest.Open(loc.Config.(rest.Config))
 	}

src/restic/backend/gcs/config.go

@@ -0,0 +1,43 @@
+package gcs
+
+import (
+	"errors"
+	"path"
+	"restic/backend/s3"
+	"strings"
+)
+
+// The endpoint for all GCS operations.
+const gcsEndpoint = "storage.googleapis.com"
+
+const defaultPrefix = "restic"
+
+// ParseConfig parses the string s and extracts the gcs config. The two
+// supported configuration formats are gcs://bucketname/prefix and
+// gcs:bucketname/prefix.
+func ParseConfig(s string) (interface{}, error) {
+	switch {
+	case strings.HasPrefix(s, "gs://"):
+		s = s[5:]
+	case strings.HasPrefix(s, "gs:"):
+		s = s[3:]
+	default:
+		return nil, errors.New(`gcs: config does not start with "gs"`)
+	}
+	p := strings.SplitN(s, "/", 2)
+	var prefix string
+	switch {
+	case len(p) < 1:
+		return nil, errors.New("gcs: invalid format: bucket name not found")
+	case len(p) == 1 || p[1] == "":
+		prefix = defaultPrefix
+	default:
+		prefix = path.Clean(p[1])
+	}
+	return s3.Config{
+		Endpoint: gcsEndpoint,
+		UseHTTP:  false,
+		Bucket:   p[0],
+		Prefix:   prefix,
+	}, nil
+}

src/restic/backend/gcs/config_test.go

@@ -0,0 +1,68 @@
+package gcs
+
+import (
+	"restic/backend/s3"
+	"testing"
+)
+
+var configTests = []struct {
+	s   string
+	cfg s3.Config
+}{
+	{"gs://bucketname", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"gs://bucketname/", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"gs://bucketname/prefix/dir", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/dir",
+	}},
+	{"gs://bucketname/prefix/dir/", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/dir",
+	}},
+	{"gs:bucketname", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"gs:bucketname/", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"gs:bucketname/prefix/dir", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/dir",
+	}},
+	{"gs:bucketname/prefix/dir/", s3.Config{
+		Endpoint: "storage.googleapis.com",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/dir",
+	}},
+}
+
+func TestParseConfig(t *testing.T) {
+	for i, test := range configTests {
+		cfg, err := ParseConfig(test.s)
+		if err != nil {
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
+			continue
+		}
+
+		if cfg != test.cfg {
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
+				i, test.s, test.cfg, cfg)
+			continue
+		}
+	}
+}

src/restic/location/location.go

@@ -4,6 +4,7 @@ package location
 import (
 	"strings"
 
+	"restic/backend/gcs"
 	"restic/backend/local"
 	"restic/backend/rest"
 	"restic/backend/s3"
@@ -28,6 +29,7 @@ var parsers = []parser{
 	{"local", local.ParseConfig},
 	{"sftp", sftp.ParseConfig},
 	{"s3", s3.ParseConfig},
+	{"gs", gcs.ParseConfig},
 	{"rest", rest.ParseConfig},
 }
 

src/restic/location/location_test.go

@@ -54,7 +54,34 @@ var parseTests = []struct {
 			Host: "host",
 			Dir:  "/srv/repo",
 		}}},
-
+	{"gs://bucketname", Location{Scheme: "gs",
+		Config: s3.Config{
+			Endpoint: "storage.googleapis.com",
+			Bucket:   "bucketname",
+			Prefix:   "restic",
+		}},
+	},
+	{"gs://bucketname/prefix", Location{Scheme: "gs",
+		Config: s3.Config{
+			Endpoint: "storage.googleapis.com",
+			Bucket:   "bucketname",
+			Prefix:   "prefix",
+		}},
+	},
+	{"gs:bucketname", Location{Scheme: "gs",
+		Config: s3.Config{
+			Endpoint: "storage.googleapis.com",
+			Bucket:   "bucketname",
+			Prefix:   "restic",
+		}},
+	},
+	{"gs:bucketname/prefix", Location{Scheme: "gs",
+		Config: s3.Config{
+			Endpoint: "storage.googleapis.com",
+			Bucket:   "bucketname",
+			Prefix:   "prefix",
+		}},
+	},
 	{"s3://eu-central-1/bucketname", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "eu-central-1",