From d94104f4ef88200f77ce0b4a4eefe9a6c4f5d0b6 Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Tue, 22 Mar 2022 16:21:49 +0300 Subject: [PATCH] feat(kube-run): --auto-pull-secret provides private registry access for pod Signed-off-by: Ilya Lesikov --- cmd/werf/kube_run/kube_run.go | 203 +++++++++++++++++++++++++++++++--- 1 file changed, 187 insertions(+), 16 deletions(-) diff --git a/cmd/werf/kube_run/kube_run.go b/cmd/werf/kube_run/kube_run.go index 9ec244ef1b..29a5a9e194 100644 --- a/cmd/werf/kube_run/kube_run.go +++ b/cmd/werf/kube_run/kube_run.go @@ -2,12 +2,17 @@ package kube_run import ( "context" + "encoding/base64" + "encoding/json" "fmt" "math/rand" "os" "os/exec" "strings" + "github.com/containers/image/v5/docker/reference" + config2 "github.com/containers/image/v5/pkg/docker/config" + imgtypes "github.com/containers/image/v5/types" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +45,7 @@ type cmdDataType struct { AllocateTty bool Rm bool RmWithNamespace bool + AutoPullSecret bool Pod string Command []string @@ -53,6 +59,15 @@ var ( commonCmdData common.CmdData ) +type dockerConfigJson struct { + Auths map[string]dockerAuthJson `json:"auths"` +} + +type dockerAuthJson struct { + Auth string `json:"auth,omitempty"` + IdentityToken string `json:"identitytoken,omitempty"` +} + func NewCmd() *cobra.Command { cmd := &cobra.Command{ Use: "kube-run [options] [IMAGE_NAME] [-- COMMAND ARG...]", @@ -153,10 +168,11 @@ func NewCmd() *cobra.Command { cmd.Flags().StringVarP(&cmdData.Pod, "pod", "", os.Getenv("WERF_POD"), "Set created pod name (default $WERF_POD or autogenerated if not specified)") cmd.Flags().StringVarP(&cmdData.Overrides, "overrides", "", os.Getenv("WERF_OVERRIDES"), "Inline JSON to override/extend any fields in created Pod, e.g. to add imagePullSecrets field (default $WERF_OVERRIDES)") cmd.Flags().StringVarP(&cmdData.ExtraOptions, "extra-options", "", os.Getenv("WERF_EXTRA_OPTIONS"), "Pass extra options to \"kubectl run\" command (default $WERF_EXTRA_OPTIONS)") - cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod after command completion (default $WERF_RM or true if not specified)") + cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod and other created resources after command completion (default $WERF_RM or true if not specified)") cmd.Flags().BoolVarP(&cmdData.RmWithNamespace, "rm-with-namespace", "", common.GetBoolEnvironmentDefaultFalse("WERF_RM_WITH_NAMESPACE"), "Remove also a namespace after command completion (default $WERF_RM_WITH_NAMESPACE or false if not specified)") cmd.Flags().BoolVarP(&cmdData.Interactive, "interactive", "i", common.GetBoolEnvironmentDefaultFalse("WERF_INTERACTIVE"), "Enable interactive mode (default $WERF_INTERACTIVE or false if not specified)") cmd.Flags().BoolVarP(&cmdData.AllocateTty, "tty", "t", common.GetBoolEnvironmentDefaultFalse("WERF_TTY"), "Allocate a TTY (default $WERF_TTY or false if not specified)") + cmd.Flags().BoolVarP(&cmdData.AutoPullSecret, "auto-pull-secret", "", common.GetBoolEnvironmentDefaultTrue("WERF_AUTO_PULL_SECRET"), "Automatically create docker config secret in the namespace and plug it via pod's imagePullSecrets for private registry access (default $WERF_AUTO_PULL_SECRET or true if not specified)") return cmd } @@ -252,6 +268,7 @@ func runMain(ctx context.Context) error { } else { pod = cmdData.Pod } + secret := pod _, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, false)) if err != nil { @@ -271,23 +288,23 @@ func runMain(ctx context.Context) error { } defer func() { - cleanupResources(ctx, pod, namespace) + cleanupResources(ctx, pod, secret, namespace) }() if *commonCmdData.Follow { return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error { - cleanupResources(ctx, pod, namespace) + cleanupResources(ctx, pod, secret, namespace) - if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil { + if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil { return err } - cleanupResources(ctx, pod, namespace) + cleanupResources(ctx, pod, secret, namespace) return nil }) } else { - if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil { + if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil { return err } } @@ -295,7 +312,7 @@ func runMain(ctx context.Context) error { return nil } -func run(ctx context.Context, pod string, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error { +func run(ctx context.Context, pod, secret, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error { projectName := werfConfig.Meta.Project projectTmpDir, err := tmp_manager.CreateProjectDir(ctx) @@ -371,6 +388,19 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W return err } + if cmdData.AutoPullSecret { + namedRef, dockerAuthConf, err := getDockerConfigCredentials(image) + if err != nil { + return fmt.Errorf("unable to get docker config credentials: %w", err) + } + + if dockerAuthConf != (imgtypes.DockerAuthConfig{}) { + if err := createDockerRegistrySecret(ctx, secret, namespace, *namedRef, dockerAuthConf); err != nil { + return fmt.Errorf("unable to create docker registry secret: %w", err) + } + } + } + args := []string{ "kubectl", "run", @@ -391,8 +421,26 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W args = append(args, "-t") } + var overrides map[string]interface{} if cmdData.Overrides != "" { - args = append(args, "--overrides", cmdData.Overrides) + if err := json.Unmarshal([]byte(cmdData.Overrides), &overrides); err != nil { + return fmt.Errorf("unable to unmarshal --overrides: %w", err) + } + } + + if cmdData.AutoPullSecret { + overrides, err = addImagePullSecret(secret, overrides) + if err != nil { + return fmt.Errorf("unable to add imagePullSecret to --overrides: %w", err) + } + } + + if len(overrides) > 0 { + overridesB, err := json.Marshal(overrides) + if err != nil { + return fmt.Errorf("unable to marshal generated --overrides: %w", err) + } + args = append(args, "--overrides", string(overridesB)) } if cmdData.ExtraOptions != "" { @@ -431,7 +479,11 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W }) } -func cleanupResources(ctx context.Context, pod string, namespace string) { +func cleanupResources(ctx context.Context, pod, secret, namespace string) { + if !cmdData.Rm { + return + } + if isNsExist, err := isNamespaceExist(ctx, namespace); err != nil { logboek.Context(ctx).Warn().LogF("WARNING: unable to check for namespace existence: %s\n", err) return @@ -439,13 +491,22 @@ func cleanupResources(ctx context.Context, pod string, namespace string) { return } - if cmdData.Rm { - if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err) - } else if isPodExist { - logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod) - if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err) + if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err) + } else if isPodExist { + logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod) + if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err) + } + } + + if cmdData.AutoPullSecret { + if isSecretExist, err := isSecretExist(ctx, secret, namespace); err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: unable to check for secret existence: %s\n", err) + } else if isSecretExist { + logboek.Context(ctx).LogF("Cleaning up secret %q ...\n", secret) + if err := kube.Client.CoreV1().Secrets(namespace).Delete(ctx, secret, v1.DeleteOptions{}); err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: secret cleaning up failed: %s\n", err) } } } @@ -480,6 +541,45 @@ func createNamespace(ctx context.Context, namespace string) error { return nil } +func createDockerRegistrySecret(ctx context.Context, name, namespace string, ref reference.Named, dockerAuthConf imgtypes.DockerAuthConfig) error { + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{}, + Type: corev1.SecretTypeDockerConfigJson, + } + + var authJson dockerAuthJson + switch { + case dockerAuthConf.IdentityToken != "": + authJson.IdentityToken = dockerAuthConf.IdentityToken + case dockerAuthConf.Username != "" && dockerAuthConf.Password != "": + authJson.Auth = base64.StdEncoding.EncodeToString([]byte(dockerAuthConf.Username + ":" + dockerAuthConf.Password)) + default: + panic("unexpected dockerAuthConf") + } + + dockerConfJson := &dockerConfigJson{ + Auths: map[string]dockerAuthJson{ + ref.Name(): authJson, + }, + } + + dockerConf, err := json.Marshal(dockerConfJson) + if err != nil { + return fmt.Errorf("unable to marshal docker config json: %w", err) + } + + secret.Data[corev1.DockerConfigJsonKey] = dockerConf + + logboek.Context(ctx).LogF("Creating secret %q in namespace %q ...\n", name, namespace) + kube.Client.CoreV1().Secrets(namespace).Create(ctx, secret, v1.CreateOptions{}) + + return nil +} + func isNamespaceExist(ctx context.Context, namespace string) (bool, error) { if matchedNamespaces, err := kube.Client.CoreV1().Namespaces().List(ctx, v1.ListOptions{ FieldSelector: fields.OneTermEqualSelector("metadata.name", namespace).String(), @@ -503,3 +603,74 @@ func isPodExist(ctx context.Context, pod string, namespace string) (bool, error) return false, nil } + +func isSecretExist(ctx context.Context, secret string, namespace string) (bool, error) { + if matchedSecrets, err := kube.Client.CoreV1().Secrets(namespace).List(ctx, v1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", secret).String(), + }); err != nil { + return false, fmt.Errorf("unable to list secrets: %w", err) + } else if len(matchedSecrets.Items) > 0 { + return true, nil + } + + return false, nil +} + +// Might return empty DockerAuthConfig. +func getDockerConfigCredentials(ref string) (*reference.Named, imgtypes.DockerAuthConfig, error) { + namedRef, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to parse docker config registry reference %q: %w", ref, err) + } + + dockerAuthConf, err := config2.GetCredentialsForRef(&imgtypes.SystemContext{AuthFilePath: *commonCmdData.DockerConfig}, namedRef) + if err != nil { + return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to get docker registry creds for ref %q: %w", ref, err) + } + + return &namedRef, dockerAuthConf, nil +} + +func addImagePullSecret(secret string, overrides map[string]interface{}) (map[string]interface{}, error) { + if secret == "" { + panic("secret name can't be empty") + } + + newImagePullSecret := map[string]interface{}{"name": secret} + newImagePullSecrets := []interface{}{newImagePullSecret} + newSpec := map[string]interface{}{"imagePullSecrets": newImagePullSecrets} + newOverrides := map[string]interface{}{"spec": newSpec} + + if len(overrides) == 0 { + return newOverrides, nil + } + + if _, ok := overrides["spec"]; !ok { + overrides["spec"] = newSpec + return overrides, nil + } + + overridesSpec, ok := overrides["spec"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected pod spec overrides format: %+v", overrides) + } + + if len(overridesSpec) == 0 { + overrides["spec"] = newSpec + return overrides, nil + } + + _, ok = overridesSpec["imagePullSecrets"] + if !ok { + overrides["spec"].(map[string]interface{})["imagePullSecrets"] = newImagePullSecrets + return overrides, nil + } + + _, ok = overridesSpec["imagePullSecrets"].([]interface{}) + if !ok { + return nil, fmt.Errorf("unexpected imagePullSecrets overrides format: %+v", overrides) + } + + overrides["spec"].(map[string]interface{})["imagePullSecrets"] = append(overrides["spec"].(map[string]interface{})["imagePullSecrets"].([]interface{}), newImagePullSecret) + return overrides, nil +}