Skip to content

Commit

Permalink
feat(secrets): preserve comments, order and aliases in the secrets ed…
Browse files Browse the repository at this point in the history
…it commands

Use yaml-v3 node-based lowlevel yaml parser to work with secret values.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Jun 21, 2022
1 parent dba573f commit 5bc6092
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 163 deletions.
119 changes: 2 additions & 117 deletions cmd/werf/helm/secret/common/edit.go
Expand Up @@ -9,13 +9,11 @@ import (
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strings"

uuid "github.com/satori/go.uuid"
"golang.org/x/crypto/ssh/terminal"
"gopkg.in/yaml.v2"

"github.com/werf/logboek"
"github.com/werf/logboek/pkg/style"
Expand Down Expand Up @@ -84,9 +82,9 @@ func SecretEdit(ctx context.Context, m *secrets_manager.SecretsManager, workingD

if !bytes.Equal(data, newData) {
if values {
newEncodedData, err = prepareResultValuesData(data, encodedData, newData, newEncodedData)
newEncodedData, err = secret.MergeEncodedYaml(data, newData, encodedData, newEncodedData)
if err != nil {
return err
return fmt.Errorf("unable to merge changed values of encoded yaml: %w", err)
}
}

Expand Down Expand Up @@ -219,116 +217,3 @@ func editor() (string, []string, error) {

return "", editorArgs, fmt.Errorf("editor not detected")
}

func prepareResultValuesData(data, encodedData, newData, newEncodedData []byte) ([]byte, error) {
dataConfig, err := unmarshalYaml(data)
if err != nil {
return nil, err
}

encodeDataConfig, err := unmarshalYaml(encodedData)
if err != nil {
return nil, err
}

newDataConfig, err := unmarshalYaml(newData)
if err != nil {
return nil, err
}

newEncodedDataConfig, err := unmarshalYaml(newEncodedData)
if err != nil {
return nil, err
}

resultEncodedDataConfig, err := mergeYamlEncodedData(dataConfig, encodeDataConfig, newDataConfig, newEncodedDataConfig)
if err != nil {
return nil, err
}

resultEncodedData, err := yaml.Marshal(&resultEncodedDataConfig)
if err != nil {
return nil, err
}

return resultEncodedData, nil
}

func unmarshalYaml(data []byte) (yaml.MapSlice, error) {
config := make(yaml.MapSlice, 0)
err := yaml.UnmarshalStrict(data, &config)
if err != nil {
return nil, err
}

return config, nil
}

func mergeYamlEncodedData(d, eD, newD, newED interface{}) (interface{}, error) {
dType := reflect.TypeOf(d)
newDType := reflect.TypeOf(newD)

if dType != newDType {
return newED, nil
}

switch newD := newD.(type) {
case yaml.MapSlice:
newDMapSlice := newD
dMapSlice := d.(yaml.MapSlice)
resultMapSlice := make(yaml.MapSlice, len(newDMapSlice))

findDMapItemByKey := func(key interface{}) (int, *yaml.MapItem) {
for ind, elm := range dMapSlice {
if elm.Key == key {
return ind, &elm
}
}

return 0, nil
}

for ind, elm := range newDMapSlice {
newEDMapItem := newED.(yaml.MapSlice)[ind]
resultMapItem := newEDMapItem

dInd, dElm := findDMapItemByKey(elm.Key)
if dElm != nil {
eDMapItem := eD.(yaml.MapSlice)[dInd]
result, err := mergeYamlEncodedData(dMapSlice[dInd], eDMapItem, newDMapSlice[ind], newEDMapItem)
if err != nil {
return nil, err
}

resultMapItem = result.(yaml.MapItem)
}

resultMapSlice[ind] = resultMapItem
}

return resultMapSlice, nil
case yaml.MapItem:
var resultMapItem yaml.MapItem
newDMapItem := newD
newEDMapItem := newED.(yaml.MapItem)
dMapItem := d.(yaml.MapItem)
eDMapItem := eD.(yaml.MapItem)

resultMapItem.Key = newDMapItem.Key

resultValue, err := mergeYamlEncodedData(dMapItem.Value, eDMapItem.Value, newDMapItem.Value, newEDMapItem.Value)
if err != nil {
return nil, err
}

resultMapItem.Value = resultValue

return resultMapItem, nil
default:
if !reflect.DeepEqual(d, newD) {
return newED, nil
} else {
return eD, nil
}
}
}
106 changes: 60 additions & 46 deletions pkg/secret/yaml_encoder.go
@@ -1,9 +1,10 @@
package secret

import (
"bytes"
"fmt"

"gopkg.in/yaml.v2"
yaml_v3 "gopkg.in/yaml.v3"
)

// YamlEncoder is an Encoder compatible object with additional helpers to work with yaml data: EncryptYamlData and DecryptYamlData
Expand Down Expand Up @@ -38,7 +39,7 @@ func (s *YamlEncoder) Encrypt(data []byte) ([]byte, error) {
}

func (s *YamlEncoder) EncryptYamlData(data []byte) ([]byte, error) {
resultData, err := doYamlData(s.generateFunc, data)
resultData, err := doYamlDataV2(s.generateFunc, data)
if err != nil {
return nil, fmt.Errorf("encryption failed: check encryption key and data: %w", err)
}
Expand All @@ -60,7 +61,7 @@ func (s *YamlEncoder) Decrypt(data []byte) ([]byte, error) {
}

func (s *YamlEncoder) DecryptYamlData(data []byte) ([]byte, error) {
resultData, err := doYamlData(s.extractFunc, data)
resultData, err := doYamlDataV2(s.extractFunc, data)
if err != nil {
if IsExtractDataError(err) {
return nil, fmt.Errorf("decryption failed: check data `%s`: %w", string(data), err)
Expand All @@ -72,74 +73,87 @@ func (s *YamlEncoder) DecryptYamlData(data []byte) ([]byte, error) {
return resultData, nil
}

func doYamlData(doFunc func([]byte) ([]byte, error), data []byte) ([]byte, error) {
config := make(yaml.MapSlice, 0)
err := yaml.UnmarshalStrict(data, &config)
if err != nil {
return nil, err
func doYamlDataV2(doFunc func([]byte) ([]byte, error), data []byte) ([]byte, error) {
var config yaml_v3.Node

if err := yaml_v3.Unmarshal(data, &config); err != nil {
return nil, fmt.Errorf("unable to unmarshal config data: %w", err)
}

resultConfig, err := doYamlValueSecret(doFunc, config)
resultConfig, err := doYamlValueSecretV2(doFunc, &config)
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to process config secrets: %w", err)
}

resultData, err := yaml.Marshal(resultConfig)
if err != nil {
return nil, err
var resultData bytes.Buffer

yamlEncoder := yaml_v3.NewEncoder(&resultData)
yamlEncoder.SetIndent(2)
if err := yamlEncoder.Encode(resultConfig); err != nil {
return nil, fmt.Errorf("unable to marshal modified config data: %w", err)
}

return resultData, nil
return resultData.Bytes(), nil
}

func doYamlValueSecret(doFunc func([]byte) ([]byte, error), data interface{}) (interface{}, error) {
switch data := data.(type) {
case yaml.MapSlice:
result := make(yaml.MapSlice, len(data))
for ind, elm := range data {
result[ind].Key = elm.Key
resultValue, err := doYamlValueSecret(doFunc, elm.Value)
func doYamlValueSecretV2(doFunc func([]byte) ([]byte, error), node *yaml_v3.Node) (*yaml_v3.Node, error) {
switch node.Kind {
case yaml_v3.DocumentNode:
for pos := 0; pos < len(node.Content); pos += 1 {
newValueNode, err := doYamlValueSecretV2(doFunc, node.Content[pos])
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to process document key %d: %w", pos, err)
}

result[ind].Value = resultValue
node.Content[pos] = newValueNode
}

return result, nil
case yaml.MapItem:
var result yaml.MapItem
case yaml_v3.MappingNode:
for pos := 0; pos < len(node.Content); pos += 2 {
keyNode := node.Content[pos]
valueNode := node.Content[pos+1]
newValueNode, err := doYamlValueSecretV2(doFunc, valueNode)
if err != nil {
return nil, fmt.Errorf("unable to process map key %q: %w", keyNode.Value, err)
}
node.Content[pos+1] = newValueNode
}

result.Key = data.Key
case yaml_v3.SequenceNode:
for pos := 0; pos < len(node.Content); pos += 1 {
newValueNode, err := doYamlValueSecretV2(doFunc, node.Content[pos])
if err != nil {
return nil, fmt.Errorf("unable to process array key %d: %w", pos, err)
}
node.Content[pos] = newValueNode
}

resultValue, err := doYamlValueSecret(doFunc, data.Value)
case yaml_v3.AliasNode:
newAliasNode, err := doYamlValueSecretV2(doFunc, node.Alias)
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to process an alias node %q: %w", node.Value, err)
}
node.Alias = newAliasNode

result.Value = resultValue
case yaml_v3.ScalarNode:
if node.ShortTag() == "!!str" {
var value string

return result, nil
case []interface{}:
var result []interface{}
for _, elm := range data {
resultElm, err := doYamlValueSecret(doFunc, elm)
if err := node.Decode(&value); err != nil {
return nil, fmt.Errorf("unable to decode string value %q: %w", node.Value, err)
}

newValue, err := doFunc([]byte(value))
if err != nil {
return nil, err
}

result = append(result, resultElm)
}

return result, nil
default:
result, err := doFunc([]byte(fmt.Sprintf("%v", data)))
if err != nil {
return nil, err
if err := node.Encode(string(newValue)); err != nil {
return nil, fmt.Errorf("unable to encode string value %q: %w", string(newValue), err)
}
}

return string(result), nil
}

return node, nil
}

func doNothing(data []byte) ([]byte, error) { return data, nil }
91 changes: 91 additions & 0 deletions pkg/secret/yaml_helpers.go
@@ -0,0 +1,91 @@
package secret

import (
"bytes"
"fmt"

yaml_v3 "gopkg.in/yaml.v3"
)

func MergeEncodedYaml(oldData, newData, oldEncodedData, newEncodedData []byte) ([]byte, error) {
var oldConfig, newConfig, oldEncodedConfig, newEncodedConfig yaml_v3.Node

for _, d := range []struct {
Data []byte
Node *yaml_v3.Node
}{
{Data: oldData, Node: &oldConfig},
{Data: newData, Node: &newConfig},
{Data: oldEncodedData, Node: &oldEncodedConfig},
{Data: newEncodedData, Node: &newEncodedConfig},
} {
if err := yaml_v3.Unmarshal(d.Data, d.Node); err != nil {
return nil, fmt.Errorf("unable to unmarshal yaml data: %w", err)
}
}

mergedNode, err := MergeEncodedYamlNode(&oldConfig, &newConfig, &oldEncodedConfig, &newEncodedConfig)
if err != nil {
return nil, err
}

var resultData bytes.Buffer

yamlEncoder := yaml_v3.NewEncoder(&resultData)
yamlEncoder.SetIndent(2)
if err := yamlEncoder.Encode(mergedNode); err != nil {
return nil, fmt.Errorf("unable to marshal merged encoded data: %w", err)
}

return resultData.Bytes(), nil
}

func MergeEncodedYamlNode(oldConfig, newConfig, oldEncodedConfig, newEncodedConfig *yaml_v3.Node) (*yaml_v3.Node, error) {
if newConfig.Kind != oldConfig.Kind {
return newEncodedConfig, nil
}

switch newEncodedConfig.Kind {
case yaml_v3.DocumentNode:
for pos := 0; pos < len(newEncodedConfig.Content); pos += 1 {
newValueNode, err := MergeEncodedYamlNode(oldConfig.Content[pos], newConfig.Content[pos], oldEncodedConfig.Content[pos], newEncodedConfig.Content[pos])
if err != nil {
return nil, fmt.Errorf("unable to process document key %d: %w", pos, err)
}
newEncodedConfig.Content[pos] = newValueNode
}

case yaml_v3.MappingNode:
for pos := 0; pos < len(newEncodedConfig.Content); pos += 2 {
newValueNode, err := MergeEncodedYamlNode(oldConfig.Content[pos+1], newConfig.Content[pos+1], oldEncodedConfig.Content[pos+1], newEncodedConfig.Content[pos+1])
if err != nil {
return nil, fmt.Errorf("unable to process map key %q: %w", newEncodedConfig.Content[pos].Value, err)
}
newEncodedConfig.Content[pos+1] = newValueNode
}

case yaml_v3.SequenceNode:
for pos := 0; pos < len(newEncodedConfig.Content); pos += 1 {
newValueNode, err := MergeEncodedYamlNode(oldConfig.Content[pos], newConfig.Content[pos], oldEncodedConfig.Content[pos], newEncodedConfig.Content[pos])
if err != nil {
return nil, fmt.Errorf("unable to process array key %d: %w", pos, err)
}
newEncodedConfig.Content[pos] = newValueNode
}

case yaml_v3.AliasNode:
newAliasNode, err := MergeEncodedYamlNode(oldConfig.Alias, newConfig.Alias, oldEncodedConfig.Alias, newEncodedConfig.Alias)
if err != nil {
return nil, fmt.Errorf("unable to process an alias node %q: %w", newEncodedConfig.Value, err)
}
newEncodedConfig.Alias = newAliasNode

case yaml_v3.ScalarNode:
if oldConfig.Value == newConfig.Value {
return oldEncodedConfig, nil
}
return newEncodedConfig, nil
}

return newEncodedConfig, nil
}

0 comments on commit 5bc6092

Please sign in to comment.