Reading GPG keys from files using environment variables (#173)

* Reading GPG keys from files using environment variables

* Little refactor

* Update crypto_comp_test.go

* Little refactor

* Fixes after review

* Update cleanup.sh

* Remove debug string

* Added README.md

* Fix tests

* Fix README.md

README.md

@@ -117,32 +117,47 @@ To enable S3 server-side encryption, set to the algorithm to use when storing th
 
 If using S3 server-side encryption with `aws:kms`, the KMS Key ID to use for object encryption.
 
-* `WALG_GPG_KEY_ID`  (alternative form `WALE_GPG_KEY_ID`)
+* `WALG_GPG_KEY_ID`  (alternative form `WALE_GPG_KEY_ID`) ⚠️ **DEPRECATED**
 
 To configure GPG key for encryption and decryption. By default, no encryption is used. Public keyring is cached in the file "/.walg_key_cache".
 
+* `WALG_PGP_KEY`
+
+To configure encryption and decryption with OpenPGP standard.
+Set *private key* value, when you need to execute ```wal-fetch``` or ```backup-fetch``` command.
+Set *public key* value, when you need to execute ```wal-push``` or ```backup-push``` command.
+Keep in mind that the *private key* also contains the *public key*.
+
+* `WALG_PGP_KEY_PATH`
+
+Similar to `WALG_PGP_KEY`, but value is the path to the key on file system.
+
+* `WALG_PGP_KEY_PASSPHRASE`
+
+If your *private key* is encrypted with a *passphrase*, you should set *passpharse* for decrypt.
+
 * `WALG_DELTA_MAX_STEPS`
 
- Delta-backup is difference between previously taken backup and present state. `WALG_DELTA_MAX_STEPS` determines how many delta backups can be between full backups. Defaults to 0.
- Restoration process will automatically fetch all necessary deltas and base backup and compose valid restored backup (you still need WALs after start of last backup to restore consistent cluster).
- Delta computation is based on ModTime of file system and LSN number of pages in datafiles.
+Delta-backup is difference between previously taken backup and present state. `WALG_DELTA_MAX_STEPS` determines how many delta backups can be between full backups. Defaults to 0.
+Restoration process will automatically fetch all necessary deltas and base backup and compose valid restored backup (you still need WALs after start of last backup to restore consistent cluster).
+Delta computation is based on ModTime of file system and LSN number of pages in datafiles.
 
 * `WALG_DELTA_ORIGIN`
 
- To configure base for next delta backup (only if `WALG_DELTA_MAX_STEPS` is not exceeded). `WALG_DELTA_ORIGIN` can be LATEST (chaining increments), LATEST_FULL (for bases where volatile part is compact and chaining has no meaning - deltas overwrite each other). Defaults to LATEST.
+To configure base for next delta backup (only if `WALG_DELTA_MAX_STEPS` is not exceeded). `WALG_DELTA_ORIGIN` can be LATEST (chaining increments), LATEST_FULL (for bases where volatile part is compact and chaining has no meaning - deltas overwrite each other). Defaults to LATEST.
 
 * `WALG_COMPRESSION_METHOD`
 
- To configure compression method used for backups. Possible options are: `lz4`, 'lzma', 'brotli'. Default method is `lz4`. LZ4 is the fastest method, but compression ratio is bad.
- LZMA is way much slower, however it compresses backups about 6 times better than LZ4. Brotli is a good trade-off between speed and compression ratio which is about 3 times better than LZ4.
+To configure compression method used for backups. Possible options are: `lz4`, 'lzma', 'brotli'. Default method is `lz4`. LZ4 is the fastest method, but compression ratio is bad.
+LZMA is way much slower, however it compresses backups about 6 times better than LZ4. Brotli is a good trade-off between speed and compression ratio which is about 3 times better than LZ4.
 
- * `WALG_DISK_RATE_LIMIT`
+* `WALG_DISK_RATE_LIMIT`
 
-  To configure disk read rate limit during ```backup-push``` in bytes per second.
+To configure disk read rate limit during ```backup-push``` in bytes per second.
 
- * `WALG_NETWORK_RATE_LIMIT`
+* `WALG_NETWORK_RATE_LIMIT`
 
-  To configure network upload rate limit during ```backup-push``` in bytes per second.
+To configure network upload rate limit during ```backup-push``` in bytes per second.
 
 Usage
 -----

cleanup.sh

@@ -2,6 +2,14 @@
 
 rm -rf tmp/
 
-if docker ps --all --format '{{.Names}}' | grep wal-g_* > /dev/null; then
-    docker rm -f $(docker ps --all --format '{{.Names}}' | grep wal-g_*)
+walg_images=$(docker ps --all --format '{{.Names}}' | grep wal-g_*)
+
+if [[ ${walg_images} ]]; then
+    docker rm -f ${walg_images}
+fi
+
+bad_images=$(docker images --filter "dangling=true" --quiet --no-trunc)
+
+if [[ ${bad_images} ]]; then
+    docker rmi ${bad_images}
 fi

docker-compose.yml

@@ -24,7 +24,11 @@ services:
       - "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE"
       - "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
     entrypoint: sh
-    command: -c 'mkdir -p /export/fullbucket && mkdir -p /export/fullscandeltabucket && mkdir -p /export/waldeltabucket && /usr/bin/minio server /export'
+    command: >
+      -c 'mkdir -p /export/fullbucket
+      &&  mkdir -p /export/fullscandeltabucket
+      &&  mkdir -p /export/waldeltabucket
+      &&  /usr/bin/minio server /export'
 
   pg:
     build:

docker/pg/walg.env

@@ -1,7 +1,7 @@
 PGSSLMODE=allow
 
-AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 AWS_ENDPOINT=http://s3:9000
 AWS_S3_FORCE_PATH_STYLE=true
 

internal/config.go

@@ -2,14 +2,13 @@ package internal
 
 import (
 	"encoding/json"
+	"github.com/go-yaml/yaml"
+	"github.com/wal-g/wal-g/internal/tracelog"
 	"io/ioutil"
 	"os"
 	"os/user"
 	"path/filepath"
 	"strings"
-
-	"github.com/go-yaml/yaml"
-	"github.com/wal-g/wal-g/internal/tracelog"
 )
 
 var (
@@ -34,6 +33,9 @@ var (
 		"WALG_S3_SSE_KMS_ID":           nil,
 		"WALG_GPG_KEY_ID":              nil,
 		"WALE_GPG_KEY_ID":              nil,
+		"WALG_PGP_KEY":                 nil,
+		"WALG_PGP_KEY_PATH":            nil,
+		"WALG_PGP_KEY_PASSPHRASE":      nil,
 		"WALG_DELTA_MAX_STEPS":         nil,
 		"WALG_DELTA_ORIGIN":            nil,
 		"WALG_COMPRESSION_METHOD":      nil,

internal/crypto.go

@@ -39,8 +39,8 @@ type CachedKey struct {
 }
 
 // TODO : unit tests
-// Here we read armoured version of Key by calling GPG process
-func getPubRingArmour(keyId string) ([]byte, error) {
+// Here we read armored version of Key by calling GPG process
+func getPubRingArmor(keyId string) ([]byte, error) {
 	var cache CachedKey
 	var cacheFilename string
 
@@ -78,7 +78,7 @@ func getPubRingArmour(keyId string) ([]byte, error) {
 	return out, nil
 }
 
-func getSecretRingArmour(keyId string) ([]byte, error) {
+func getSecretRingArmor(keyId string) ([]byte, error) {
 	out, err := exec.Command(GpgBin, "-a", "--export-secret-key", keyId).Output()
 	if err != nil {
 		return nil, err

internal/crypto_new.go

@@ -0,0 +1,56 @@
+package internal
+
+import (
+	"bytes"
+	"golang.org/x/crypto/openpgp"
+	"io"
+	"io/ioutil"
+)
+
+func ReadKey(path string) (io.Reader, error) {
+	byteData, err := ioutil.ReadFile(path)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return bytes.NewReader(byteData), nil
+}
+
+func ReadPGPKey(path string) (openpgp.EntityList, error) {
+	gpgKeyReader, err := ReadKey(path)
+
+	if err != nil {
+		return nil, err
+	}
+
+	entityList, err := openpgp.ReadArmoredKeyRing(gpgKeyReader)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return entityList, nil
+}
+
+func DecryptSecretKey(entityList openpgp.EntityList, passphrase string) error {
+	passphraseBytes := []byte(passphrase)
+
+	for _, entity := range entityList {
+		err := entity.PrivateKey.Decrypt(passphraseBytes)
+
+		if err != nil {
+			return err
+		}
+
+		for _, subKey := range entity.Subkeys {
+			err := subKey.PrivateKey.Decrypt(passphraseBytes)
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

internal/open_pgp_crypter.go

@@ -7,6 +7,7 @@ import (
 	"github.com/wal-g/wal-g/internal/tracelog"
 	"golang.org/x/crypto/openpgp"
 	"io"
+	"strings"
 )
 
 // CrypterUseMischiefError happens when crypter is used before initialization
@@ -28,47 +29,110 @@ func (err CrypterUseMischiefError) Error() string {
 // to extract interface
 type OpenPGPCrypter struct {
 	Configured bool
-	KeyRingId  string
+
+	KeyRingId      string
+	IsUseKeyRingId bool
+
+	ArmoredKey      string
+	IsUseArmoredKey bool
+
+	ArmoredKeyPath   string
+	IsUseArmoredKeyPath bool
 
 	PubKey    openpgp.EntityList
 	SecretKey openpgp.EntityList
 }
 
 func (crypter *OpenPGPCrypter) IsArmed() bool {
-	return len(crypter.KeyRingId) != 0
+	if crypter.IsUseKeyRingId {
+		tracelog.WarningLogger.Println(`
+You are using deprecated functionality that uses an external gpg library.
+It will be removed in next major version.
+Please set GPG key using environment variables WALG_PGP_KEY or WALG_PGP_KEY_PATH.
+		`)
+	}
+
+	return crypter.IsUseArmoredKey || crypter.IsUseArmoredKeyPath || crypter.IsUseKeyRingId
 }
 
 // IsUsed is to check necessity of Crypter use
 // Must be called prior to any other crypter call
 func (crypter *OpenPGPCrypter) IsUsed() bool {
 	if !crypter.Configured {
-		crypter.ConfigureGPGCrypter()
+		crypter.ConfigurePGPCrypter()
 	}
+
 	return crypter.IsArmed()
 }
 
-// ConfigureGPGCrypter is OpenPGPCrypter internal initialization
-func (crypter *OpenPGPCrypter) ConfigureGPGCrypter() {
+// OpenPGPCrypter internal initialization
+func (crypter *OpenPGPCrypter) ConfigurePGPCrypter() {
 	crypter.Configured = true
-	crypter.KeyRingId = GetKeyRingId()
+
+	// key can be either private (for download) or public (for upload)
+	armoredKey, isKeyExist := LookupConfigValue("WALG_PGP_KEY")
+
+	if isKeyExist {
+		crypter.ArmoredKey = armoredKey
+		crypter.IsUseArmoredKey = true
+
+		return
+	}
+
+	// key can be either private (for download) or public (for upload)
+	armoredKeyPath, isPathExist := LookupConfigValue("WALG_PGP_KEY_PATH")
+
+	if isPathExist {
+		crypter.ArmoredKeyPath = armoredKeyPath
+		crypter.IsUseArmoredKeyPath = true
+
+		return
+	}
+
+	if crypter.KeyRingId = GetKeyRingId(); crypter.KeyRingId != "" {
+		crypter.IsUseKeyRingId = true
+	}
 }
 
 // Encrypt creates encryption writer from ordinary writer
 func (crypter *OpenPGPCrypter) Encrypt(writer io.WriteCloser) (io.WriteCloser, error) {
 	if !crypter.Configured {
 		return nil, NewCrypterUseMischiefError()
 	}
+
 	if crypter.PubKey == nil {
-		armour, err := getPubRingArmour(crypter.KeyRingId)
-		if err != nil {
-			return nil, err
-		}
+		if crypter.IsUseArmoredKey {
+			entityList, err := openpgp.ReadArmoredKeyRing(strings.NewReader(crypter.ArmoredKey))
 
-		entitylist, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armour))
-		if err != nil {
-			return nil, err
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.PubKey = entityList
+		} else if crypter.IsUseArmoredKeyPath {
+			entityList, err := ReadPGPKey(crypter.ArmoredKeyPath)
+
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.PubKey = entityList
+		} else {
+			// TODO: legacy gpg external use, need to remove in next major version
+			armor, err := getPubRingArmor(crypter.KeyRingId)
+
+			if err != nil {
+				return nil, err
+			}
+
+			entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armor))
+
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.PubKey = entityList
 		}
-		crypter.PubKey = entitylist
 	}
 
 	return &DelayWriteCloser{writer, crypter.PubKey, nil}, nil
@@ -79,22 +143,54 @@ func (crypter *OpenPGPCrypter) Decrypt(reader io.ReadCloser) (io.Reader, error)
 	if !crypter.Configured {
 		return nil, NewCrypterUseMischiefError()
 	}
+
 	if crypter.SecretKey == nil {
-		armour, err := getSecretRingArmour(crypter.KeyRingId)
-		if err != nil {
-			return nil, err
+		if crypter.IsUseArmoredKey {
+			entityList, err := openpgp.ReadArmoredKeyRing(strings.NewReader(crypter.ArmoredKey))
+
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.SecretKey = entityList
+		} else if crypter.IsUseArmoredKeyPath {
+			entityList, err := ReadPGPKey(crypter.ArmoredKeyPath)
+
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.SecretKey = entityList
+		} else {
+			// TODO: legacy gpg external use, need to remove in next major version
+			armor, err := getSecretRingArmor(crypter.KeyRingId)
+
+			if err != nil {
+				return nil, err
+			}
+
+			entityList, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armor))
+
+			if err != nil {
+				return nil, err
+			}
+
+			crypter.SecretKey = entityList
 		}
 
-		entitylist, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armour))
-		if err != nil {
-			return nil, err
+		if passphrase, isExist := LookupConfigValue("WALG_PGP_KEY_PASSPHRASE"); isExist {
+			err := DecryptSecretKey(crypter.SecretKey, passphrase)
+
+			if err != nil {
+				return nil, err
+			}
 		}
-		crypter.SecretKey = entitylist
 	}
 
-	var md, err0 = openpgp.ReadMessage(reader, crypter.SecretKey, nil, nil)
-	if err0 != nil {
-		return nil, err0
+	md, err := openpgp.ReadMessage(reader, crypter.SecretKey, nil, nil)
+
+	if err != nil {
+		return nil, err
 	}
 
 	return md.UnverifiedBody, nil