Skip to content

Commit

Permalink
Feature/envelope pgp (#1547)
Browse files Browse the repository at this point in the history 
* feat: init wal-g with envelope gpg via yckms

* feat: yckms tests

* feat: envelop openpgp tests

* fix: comment

* fix: import name

* feat: stale cache with ttl and mockery in tests

* fix: restore after merge master

* feat: pass pgp encrypted cache expiration in allowed core keys

* fix: linter

* feat: took out pgp crypters creation

* fix: use base64 encoded encrypted key

* fix: crypto configuration

detect crypto type at first then configure particular

* fix: change enveloper signature

* feat: resetup default value for cached enveloper

* fix: confegure crypter comment

* fix: distinct yc kms key settings for encrypted gpg

* fix: add yc kms in white list keys

* fix: error text

* fix: rename env previx for envelope gpg key

* fix: encrypted -> envelope

* fix: use IsZero to check zero time

* docs: envelope gpg key

---------

Co-authored-by: Ivan Sitkin <alviner@yandex-team.ru>
  • Loading branch information
@Alviner
Alviner and Ivan Sitkin committed Sep 8, 2023
1 parent 5b6fe11 commit c20716f
Show file tree
Hide file tree
Showing 16 changed files with 1,115 additions and 113 deletions.
27 changes: 27 additions & 0 deletions docs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,33 @@ Similar to `WALG_PGP_KEY`, but value is the path to the key on file system.

If your *private key* is encrypted with a *passphrase*, you should set *passphrase* for decrypt.

* `WALG_ENVELOPE_PGP_KEY`
To configure encryption and decryption with the envelope PGP key stored in key management system.
This option allows you to securely manage your PGP keys by storing them in the KMS.
It is crucial to ensure that the key passed is encrypted using kms and encoded with *base64*.
Also both *private* and *publlic* parts should be presents in key because envelope key will be injected in metadata and used later in `wal/backup-fetch`.

Please note that currently, only Yandex Cloud Key Management Service (KMS) is supported for configuring.
Ensure that you have set up and configured Yandex Cloud KMS mentioned below before attempting to use this feature.

* `WALG_ENVELOPE_CACHE_EXPIRATION`

This setting controls kms response expiration. Default value is `0` to store keys permanent in memory.
Please note that if the system will not be able to redecrypt the key in kms after expiration, the previous response will be used.

* `WALG_ENVELOPE_PGP_YC_CSE_KMS_KEY_ID`

Similar to `YC_CSE_KMS_KEY_ID`, but only used for envelope pgp keys.

* `WALG_ENVELOPE_PGP_YC_SERVICE_ACCOUNT_KEY_FILE`

Similar to `YC_SERVICE_ACCOUNT_KEY_FILE`, but only used for envelope pgp keys.

* `WALG_ENVELOPE_PGP_KEY_PATH`

Similar to `WALG_ENVELOPE_PGP_KEY`, but value is the path to the key on file system.


### Monitoring

* `WALG_STATSD_ADDRESS`
Expand Down
171 changes: 92 additions & 79 deletions internal/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,43 +29,49 @@ const (
MONGO = "MONGO"
GP = "GP"

DownloadConcurrencySetting = "WALG_DOWNLOAD_CONCURRENCY"
UploadConcurrencySetting = "WALG_UPLOAD_CONCURRENCY"
UploadDiskConcurrencySetting = "WALG_UPLOAD_DISK_CONCURRENCY"
UploadQueueSetting = "WALG_UPLOAD_QUEUE"
SentinelUserDataSetting = "WALG_SENTINEL_USER_DATA"
PreventWalOverwriteSetting = "WALG_PREVENT_WAL_OVERWRITE"
UploadWalMetadata = "WALG_UPLOAD_WAL_METADATA"
DeltaMaxStepsSetting = "WALG_DELTA_MAX_STEPS"
DeltaOriginSetting = "WALG_DELTA_ORIGIN"
CompressionMethodSetting = "WALG_COMPRESSION_METHOD"
StoragePrefixSetting = "WALG_STORAGE_PREFIX"
DiskRateLimitSetting = "WALG_DISK_RATE_LIMIT"
NetworkRateLimitSetting = "WALG_NETWORK_RATE_LIMIT"
UseWalDeltaSetting = "WALG_USE_WAL_DELTA"
UseReverseUnpackSetting = "WALG_USE_REVERSE_UNPACK"
SkipRedundantTarsSetting = "WALG_SKIP_REDUNDANT_TARS"
VerifyPageChecksumsSetting = "WALG_VERIFY_PAGE_CHECKSUMS"
StoreAllCorruptBlocksSetting = "WALG_STORE_ALL_CORRUPT_BLOCKS"
UseRatingComposerSetting = "WALG_USE_RATING_COMPOSER"
UseCopyComposerSetting = "WALG_USE_COPY_COMPOSER"
UseDatabaseComposerSetting = "WALG_USE_DATABASE_COMPOSER"
WithoutFilesMetadataSetting = "WALG_WITHOUT_FILES_METADATA"
DeltaFromNameSetting = "WALG_DELTA_FROM_NAME"
DeltaFromUserDataSetting = "WALG_DELTA_FROM_USER_DATA"
FetchTargetUserDataSetting = "WALG_FETCH_TARGET_USER_DATA"
LogLevelSetting = "WALG_LOG_LEVEL"
TarSizeThresholdSetting = "WALG_TAR_SIZE_THRESHOLD"
TarDisableFsyncSetting = "WALG_TAR_DISABLE_FSYNC"
CseKmsIDSetting = "WALG_CSE_KMS_ID"
CseKmsRegionSetting = "WALG_CSE_KMS_REGION"
LibsodiumKeySetting = "WALG_LIBSODIUM_KEY"
LibsodiumKeyPathSetting = "WALG_LIBSODIUM_KEY_PATH"
LibsodiumKeyTransform = "WALG_LIBSODIUM_KEY_TRANSFORM"
GpgKeyIDSetting = "GPG_KEY_ID"
PgpKeySetting = "WALG_PGP_KEY"
PgpKeyPathSetting = "WALG_PGP_KEY_PATH"
PgpKeyPassphraseSetting = "WALG_PGP_KEY_PASSPHRASE"
DownloadConcurrencySetting = "WALG_DOWNLOAD_CONCURRENCY"
UploadConcurrencySetting = "WALG_UPLOAD_CONCURRENCY"
UploadDiskConcurrencySetting = "WALG_UPLOAD_DISK_CONCURRENCY"
UploadQueueSetting = "WALG_UPLOAD_QUEUE"
SentinelUserDataSetting = "WALG_SENTINEL_USER_DATA"
PreventWalOverwriteSetting = "WALG_PREVENT_WAL_OVERWRITE"
UploadWalMetadata = "WALG_UPLOAD_WAL_METADATA"
DeltaMaxStepsSetting = "WALG_DELTA_MAX_STEPS"
DeltaOriginSetting = "WALG_DELTA_ORIGIN"
CompressionMethodSetting = "WALG_COMPRESSION_METHOD"
StoragePrefixSetting = "WALG_STORAGE_PREFIX"
DiskRateLimitSetting = "WALG_DISK_RATE_LIMIT"
NetworkRateLimitSetting = "WALG_NETWORK_RATE_LIMIT"
UseWalDeltaSetting = "WALG_USE_WAL_DELTA"
UseReverseUnpackSetting = "WALG_USE_REVERSE_UNPACK"
SkipRedundantTarsSetting = "WALG_SKIP_REDUNDANT_TARS"
VerifyPageChecksumsSetting = "WALG_VERIFY_PAGE_CHECKSUMS"
StoreAllCorruptBlocksSetting = "WALG_STORE_ALL_CORRUPT_BLOCKS"
UseRatingComposerSetting = "WALG_USE_RATING_COMPOSER"
UseCopyComposerSetting = "WALG_USE_COPY_COMPOSER"
UseDatabaseComposerSetting = "WALG_USE_DATABASE_COMPOSER"
WithoutFilesMetadataSetting = "WALG_WITHOUT_FILES_METADATA"
DeltaFromNameSetting = "WALG_DELTA_FROM_NAME"
DeltaFromUserDataSetting = "WALG_DELTA_FROM_USER_DATA"
FetchTargetUserDataSetting = "WALG_FETCH_TARGET_USER_DATA"
LogLevelSetting = "WALG_LOG_LEVEL"
TarSizeThresholdSetting = "WALG_TAR_SIZE_THRESHOLD"
TarDisableFsyncSetting = "WALG_TAR_DISABLE_FSYNC"
CseKmsIDSetting = "WALG_CSE_KMS_ID"
CseKmsRegionSetting = "WALG_CSE_KMS_REGION"
LibsodiumKeySetting = "WALG_LIBSODIUM_KEY"
LibsodiumKeyPathSetting = "WALG_LIBSODIUM_KEY_PATH"
LibsodiumKeyTransform = "WALG_LIBSODIUM_KEY_TRANSFORM"
GpgKeyIDSetting = "GPG_KEY_ID"
PgpKeySetting = "WALG_PGP_KEY"
PgpKeyPathSetting = "WALG_PGP_KEY_PATH"
PgpKeyPassphraseSetting = "WALG_PGP_KEY_PASSPHRASE"
PgpEnvelopeKeySetting = "WALG_ENVELOPE_PGP_KEY"
PgpEnvelopKeyPathSetting = "WALG_ENVELOPE_PGP_KEY_PATH"
PgpEnvelopeYcKmsKeyIDSetting = "WALG_ENVELOPE_PGP_YC_CSE_KMS_KEY_ID"
PgpEnvelopeYcSaKeyFileSetting = "WALG_ENVELOPE_PGP_YC_SERVICE_ACCOUNT_KEY_FILE"
PgpEnvelopeCacheExpiration = "WALG_ENVELOPE_CACHE_EXPIRATION"

PgDataSetting = "PGDATA"
UserSetting = "USER" // TODO : do something with it
PgPortSetting = "PGPORT"
Expand Down Expand Up @@ -219,6 +225,7 @@ var (
LibsodiumKeyTransform: "none",
PgFailoverStoragesCheckTimeout: "30s",
PgFailoverStorageCacheLifetime: "15m",
PgpEnvelopeCacheExpiration: "0",
}

MongoDefaultSettings = map[string]string{
Expand Down Expand Up @@ -263,48 +270,53 @@ var (

CommonAllowedSettings = map[string]bool{
// WAL-G core
DownloadConcurrencySetting: true,
UploadConcurrencySetting: true,
UploadDiskConcurrencySetting: true,
UploadQueueSetting: true,
SentinelUserDataSetting: true,
PreventWalOverwriteSetting: true,
UploadWalMetadata: true,
DeltaMaxStepsSetting: true,
DeltaOriginSetting: true,
CompressionMethodSetting: true,
StoragePrefixSetting: true,
DiskRateLimitSetting: true,
NetworkRateLimitSetting: true,
UseWalDeltaSetting: true,
LogLevelSetting: true,
TarSizeThresholdSetting: true,
TarDisableFsyncSetting: true,
"WALG_" + GpgKeyIDSetting: true,
"WALE_" + GpgKeyIDSetting: true,
PgpKeySetting: true,
PgpKeyPathSetting: true,
PgpKeyPassphraseSetting: true,
LibsodiumKeySetting: true,
LibsodiumKeyPathSetting: true,
LibsodiumKeyTransform: true,
TotalBgUploadedLimit: true,
NameStreamCreateCmd: true,
NameStreamRestoreCmd: true,
UseReverseUnpackSetting: true,
SkipRedundantTarsSetting: true,
VerifyPageChecksumsSetting: true,
StoreAllCorruptBlocksSetting: true,
UseRatingComposerSetting: true,
UseCopyComposerSetting: true,
UseDatabaseComposerSetting: true,
WithoutFilesMetadataSetting: true,
MaxDelayedSegmentsCount: true,
DeltaFromNameSetting: true,
DeltaFromUserDataSetting: true,
FetchTargetUserDataSetting: true,
SerializerTypeSetting: true,
StatsdAddressSetting: true,
DownloadConcurrencySetting: true,
UploadConcurrencySetting: true,
UploadDiskConcurrencySetting: true,
UploadQueueSetting: true,
SentinelUserDataSetting: true,
PreventWalOverwriteSetting: true,
UploadWalMetadata: true,
DeltaMaxStepsSetting: true,
DeltaOriginSetting: true,
CompressionMethodSetting: true,
StoragePrefixSetting: true,
DiskRateLimitSetting: true,
NetworkRateLimitSetting: true,
UseWalDeltaSetting: true,
LogLevelSetting: true,
TarSizeThresholdSetting: true,
TarDisableFsyncSetting: true,
"WALG_" + GpgKeyIDSetting: true,
"WALE_" + GpgKeyIDSetting: true,
PgpKeySetting: true,
PgpKeyPathSetting: true,
PgpKeyPassphraseSetting: true,
PgpEnvelopeKeySetting: true,
PgpEnvelopKeyPathSetting: true,
PgpEnvelopeCacheExpiration: true,
PgpEnvelopeYcKmsKeyIDSetting: true,
PgpEnvelopeYcSaKeyFileSetting: true,
LibsodiumKeySetting: true,
LibsodiumKeyPathSetting: true,
LibsodiumKeyTransform: true,
TotalBgUploadedLimit: true,
NameStreamCreateCmd: true,
NameStreamRestoreCmd: true,
UseReverseUnpackSetting: true,
SkipRedundantTarsSetting: true,
VerifyPageChecksumsSetting: true,
StoreAllCorruptBlocksSetting: true,
UseRatingComposerSetting: true,
UseCopyComposerSetting: true,
UseDatabaseComposerSetting: true,
WithoutFilesMetadataSetting: true,
MaxDelayedSegmentsCount: true,
DeltaFromNameSetting: true,
DeltaFromUserDataSetting: true,
FetchTargetUserDataSetting: true,
SerializerTypeSetting: true,
StatsdAddressSetting: true,

ProfileSamplingRatio: true,
ProfileMode: true,
Expand Down Expand Up @@ -496,6 +508,7 @@ var (
PgPasswordSetting: true,
PgpKeyPassphraseSetting: true,
PgpKeySetting: true,
PgpEnvelopeKeySetting: true,
RedisPassword: true,
SQLServerConnectionString: true,
SSHPassword: true,
Expand Down
90 changes: 74 additions & 16 deletions internal/configure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ import (
"github.com/wal-g/wal-g/internal/compression"
"github.com/wal-g/wal-g/internal/crypto"
"github.com/wal-g/wal-g/internal/crypto/awskms"
cachenvlpr "github.com/wal-g/wal-g/internal/crypto/envelope/enveloper/cached"
yckmsenvlpr "github.com/wal-g/wal-g/internal/crypto/envelope/enveloper/yckms"
envopenpgp "github.com/wal-g/wal-g/internal/crypto/envelope/openpgp"
"github.com/wal-g/wal-g/internal/crypto/openpgp"
"github.com/wal-g/wal-g/internal/fsutil"
"github.com/wal-g/wal-g/internal/limiters"
Expand Down Expand Up @@ -281,7 +284,11 @@ func ConfigureSplitUploader() (Uploader, error) {
}

func ConfigureCrypter() crypto.Crypter {
return ConfigureCrypterForSpecificConfig(viper.GetViper())
crypter, err := ConfigureCrypterForSpecificConfig(viper.GetViper())
if err != nil {
tracelog.ErrorLogger.FatalfOnError("can't configure crypter: %v", err)
}
return crypter
}

func CrypterFromConfig(configFile string) crypto.Crypter {
Expand All @@ -290,44 +297,95 @@ func CrypterFromConfig(configFile string) crypto.Crypter {
ReadConfigFromFile(config, configFile)
CheckAllowedSettings(config)

return ConfigureCrypterForSpecificConfig(config)
crypter, err := ConfigureCrypterForSpecificConfig(config)
if err != nil {
tracelog.ErrorLogger.FatalfOnError("can't configure crypter: %v", err)
}
return crypter
}

// ConfigureCrypter uses environment variables to create and configure a crypter.
// In case no configuration in environment variables found, return `<nil>` value.
func ConfigureCrypterForSpecificConfig(config *viper.Viper) crypto.Crypter {
// In case no configuration in environment variables found, return `<nil>` crypter.
func ConfigureCrypterForSpecificConfig(config *viper.Viper) (crypto.Crypter, error) {
pgpKey := config.IsSet(PgpKeySetting)
pgpKeyPath := config.IsSet(PgpKeyPathSetting)
legacyGpg := config.IsSet(GpgKeyIDSetting)

envelopePgpKey := config.IsSet(PgpEnvelopKeyPathSetting)
envelopePgpKeyPath := config.IsSet(PgpEnvelopeKeySetting)

libsodiumKey := config.IsSet(LibsodiumKeySetting)
libsodiumKeyPath := config.IsSet(LibsodiumKeySetting)

isPgpKey := pgpKey || pgpKeyPath || legacyGpg
isEnvelopePgpKey := envelopePgpKey || envelopePgpKeyPath
isLibsodium := libsodiumKey || libsodiumKeyPath

if isPgpKey && isEnvelopePgpKey {
return nil, errors.New("there is no way to configure plain gpg and envelope gpg at the same time, please choose one")
}

switch {
case isPgpKey:
return configurePgpCrypter(config)
case isEnvelopePgpKey:
return configureEnvelopePgpCrypter(config)
case config.IsSet(CseKmsIDSetting):
return awskms.CrypterFromKeyID(config.GetString(CseKmsIDSetting), config.GetString(CseKmsRegionSetting)), nil
case config.IsSet(YcKmsKeyIDSetting):
return yckms.YcCrypterFromKeyIDAndCredential(config.GetString(YcKmsKeyIDSetting), config.GetString(YcSaKeyFileSetting)), nil
case isLibsodium:
return configureLibsodiumCrypter(config)
default:
return nil, nil
}
}

func configurePgpCrypter(config *viper.Viper) (crypto.Crypter, error) {
loadPassphrase := func() (string, bool) {
return GetSetting(PgpKeyPassphraseSetting)
}

// key can be either private (for download) or public (for upload)
if config.IsSet(PgpKeySetting) {
return openpgp.CrypterFromKey(config.GetString(PgpKeySetting), loadPassphrase)
return openpgp.CrypterFromKey(config.GetString(PgpKeySetting), loadPassphrase), nil
}

// key can be either private (for download) or public (for upload)
if config.IsSet(PgpKeyPathSetting) {
return openpgp.CrypterFromKeyPath(config.GetString(PgpKeyPathSetting), loadPassphrase)
return openpgp.CrypterFromKeyPath(config.GetString(PgpKeyPathSetting), loadPassphrase), nil
}

if keyRingID, ok := getWaleCompatibleSetting(GpgKeyIDSetting); ok {
tracelog.WarningLogger.Printf(DeprecatedExternalGpgMessage)
return openpgp.CrypterFromKeyRingID(keyRingID, loadPassphrase)
return openpgp.CrypterFromKeyRingID(keyRingID, loadPassphrase), nil
}
return nil, errors.New("there is no any supported gpg crypter configuration")
}

if config.IsSet(CseKmsIDSetting) {
return awskms.CrypterFromKeyID(config.GetString(CseKmsIDSetting), config.GetString(CseKmsRegionSetting))
func configureEnvelopePgpCrypter(config *viper.Viper) (crypto.Crypter, error) {
if !config.IsSet(PgpEnvelopeYcKmsKeyIDSetting) {
return nil, errors.New("yandex cloud KMS key for client-side encryption and decryption must be configured")
}

if config.IsSet(YcKmsKeyIDSetting) {
return yckms.YcCrypterFromKeyIDAndCredential(config.GetString(YcKmsKeyIDSetting), config.GetString(YcSaKeyFileSetting))
yckmsEnveloper, err := yckmsenvlpr.EnveloperFromKeyIDAndCredential(
config.GetString(PgpEnvelopeYcKmsKeyIDSetting), config.GetString(PgpEnvelopeYcSaKeyFileSetting),
)
if err != nil {
return nil, err
}

if crypter := configureLibsodiumCrypter(); crypter != nil {
return crypter
expiration, err := GetDurationSetting(PgpEnvelopeCacheExpiration)
if err != nil {
return nil, err
}
enveloper := cachenvlpr.EnveloperWithCache(yckmsEnveloper, expiration)

return nil
if config.IsSet(PgpEnvelopKeyPathSetting) {
return envopenpgp.CrypterFromKeyPath(viper.GetString(PgpEnvelopKeyPathSetting), enveloper), nil
}
if config.IsSet(PgpEnvelopeKeySetting) {
return envopenpgp.CrypterFromKey(viper.GetString(PgpEnvelopeKeySetting), enveloper), nil
}
return nil, errors.New("there is no any supported envelope gpg crypter configuration")
}

func GetMaxDownloadConcurrency() (int, error) {
Expand Down

0 comments on commit c20716f

Please sign in to comment.