Skip to content

Commit

Permalink
feat(kube-run): --auto-pull-secret provides private registry access f…
Browse files Browse the repository at this point in the history 
…or pod

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>
  • Loading branch information
@ilya-lesikov
ilya-lesikov committed Mar 22, 2022
1 parent 352a0bd commit d94104f
Showing 1 changed file with 187 additions and 16 deletions.
203 changes: 187 additions & 16 deletions cmd/werf/kube_run/kube_run.go
View file
Expand Up @@ -2,12 +2,17 @@ package kube_run

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"math/rand"
"os"
"os/exec"
"strings"

"github.com/containers/image/v5/docker/reference"
config2 "github.com/containers/image/v5/pkg/docker/config"
imgtypes "github.com/containers/image/v5/types"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -40,6 +45,7 @@ type cmdDataType struct {
AllocateTty bool
Rm bool
RmWithNamespace bool
AutoPullSecret bool

Pod string
Command []string
Expand All @@ -53,6 +59,15 @@ var (
commonCmdData common.CmdData
)

type dockerConfigJson struct {
Auths map[string]dockerAuthJson `json:"auths"`
}

type dockerAuthJson struct {
Auth string `json:"auth,omitempty"`
IdentityToken string `json:"identitytoken,omitempty"`
}

func NewCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "kube-run [options] [IMAGE_NAME] [-- COMMAND ARG...]",
Expand Down Expand Up @@ -153,10 +168,11 @@ func NewCmd() *cobra.Command {
cmd.Flags().StringVarP(&cmdData.Pod, "pod", "", os.Getenv("WERF_POD"), "Set created pod name (default $WERF_POD or autogenerated if not specified)")
cmd.Flags().StringVarP(&cmdData.Overrides, "overrides", "", os.Getenv("WERF_OVERRIDES"), "Inline JSON to override/extend any fields in created Pod, e.g. to add imagePullSecrets field (default $WERF_OVERRIDES)")
cmd.Flags().StringVarP(&cmdData.ExtraOptions, "extra-options", "", os.Getenv("WERF_EXTRA_OPTIONS"), "Pass extra options to \"kubectl run\" command (default $WERF_EXTRA_OPTIONS)")
cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod after command completion (default $WERF_RM or true if not specified)")
cmd.Flags().BoolVarP(&cmdData.Rm, "rm", "", common.GetBoolEnvironmentDefaultTrue("WERF_RM"), "Remove pod and other created resources after command completion (default $WERF_RM or true if not specified)")
cmd.Flags().BoolVarP(&cmdData.RmWithNamespace, "rm-with-namespace", "", common.GetBoolEnvironmentDefaultFalse("WERF_RM_WITH_NAMESPACE"), "Remove also a namespace after command completion (default $WERF_RM_WITH_NAMESPACE or false if not specified)")
cmd.Flags().BoolVarP(&cmdData.Interactive, "interactive", "i", common.GetBoolEnvironmentDefaultFalse("WERF_INTERACTIVE"), "Enable interactive mode (default $WERF_INTERACTIVE or false if not specified)")
cmd.Flags().BoolVarP(&cmdData.AllocateTty, "tty", "t", common.GetBoolEnvironmentDefaultFalse("WERF_TTY"), "Allocate a TTY (default $WERF_TTY or false if not specified)")
cmd.Flags().BoolVarP(&cmdData.AutoPullSecret, "auto-pull-secret", "", common.GetBoolEnvironmentDefaultTrue("WERF_AUTO_PULL_SECRET"), "Automatically create docker config secret in the namespace and plug it via pod's imagePullSecrets for private registry access (default $WERF_AUTO_PULL_SECRET or true if not specified)")

return cmd
}
Expand Down Expand Up @@ -252,6 +268,7 @@ func runMain(ctx context.Context) error {
} else {
pod = cmdData.Pod
}
secret := pod

_, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, false))
if err != nil {
Expand All @@ -271,31 +288,31 @@ func runMain(ctx context.Context) error {
}

defer func() {
cleanupResources(ctx, pod, namespace)
cleanupResources(ctx, pod, secret, namespace)
}()

if *commonCmdData.Follow {
return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error {
cleanupResources(ctx, pod, namespace)
cleanupResources(ctx, pod, secret, namespace)

if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
return err
}

cleanupResources(ctx, pod, namespace)
cleanupResources(ctx, pod, secret, namespace)

return nil
})
} else {
if err := run(ctx, pod, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
if err := run(ctx, pod, secret, namespace, werfConfig, containerRuntime, giterminismManager); err != nil {
return err
}
}

return nil
}

func run(ctx context.Context, pod string, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error {
func run(ctx context.Context, pod, secret, namespace string, werfConfig *config.WerfConfig, containerRuntime container_runtime.ContainerRuntime, giterminismManager giterminism_manager.Interface) error {
projectName := werfConfig.Meta.Project

projectTmpDir, err := tmp_manager.CreateProjectDir(ctx)
Expand Down Expand Up @@ -371,6 +388,19 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
return err
}

if cmdData.AutoPullSecret {
namedRef, dockerAuthConf, err := getDockerConfigCredentials(image)
if err != nil {
return fmt.Errorf("unable to get docker config credentials: %w", err)
}

if dockerAuthConf != (imgtypes.DockerAuthConfig{}) {
if err := createDockerRegistrySecret(ctx, secret, namespace, *namedRef, dockerAuthConf); err != nil {
return fmt.Errorf("unable to create docker registry secret: %w", err)
}
}
}

args := []string{
"kubectl",
"run",
Expand All @@ -391,8 +421,26 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
args = append(args, "-t")
}

var overrides map[string]interface{}
if cmdData.Overrides != "" {
args = append(args, "--overrides", cmdData.Overrides)
if err := json.Unmarshal([]byte(cmdData.Overrides), &overrides); err != nil {
return fmt.Errorf("unable to unmarshal --overrides: %w", err)
}
}

if cmdData.AutoPullSecret {
overrides, err = addImagePullSecret(secret, overrides)
if err != nil {
return fmt.Errorf("unable to add imagePullSecret to --overrides: %w", err)
}
}

if len(overrides) > 0 {
overridesB, err := json.Marshal(overrides)
if err != nil {
return fmt.Errorf("unable to marshal generated --overrides: %w", err)
}
args = append(args, "--overrides", string(overridesB))
}

if cmdData.ExtraOptions != "" {
Expand Down Expand Up @@ -431,21 +479,34 @@ func run(ctx context.Context, pod string, namespace string, werfConfig *config.W
})
}

func cleanupResources(ctx context.Context, pod string, namespace string) {
func cleanupResources(ctx context.Context, pod, secret, namespace string) {
if !cmdData.Rm {
return
}

if isNsExist, err := isNamespaceExist(ctx, namespace); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: unable to check for namespace existence: %s\n", err)
return
} else if !isNsExist {
return
}

if cmdData.Rm {
if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err)
} else if isPodExist {
logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod)
if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err)
if isPodExist, err := isPodExist(ctx, pod, namespace); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: unable to check for pod existence: %s\n", err)
} else if isPodExist {
logboek.Context(ctx).LogF("Cleaning up pod %q ...\n", pod)
if err := kube.Client.CoreV1().Pods(namespace).Delete(ctx, pod, v1.DeleteOptions{}); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: pod cleaning up failed: %s\n", err)
}
}

if cmdData.AutoPullSecret {
if isSecretExist, err := isSecretExist(ctx, secret, namespace); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: unable to check for secret existence: %s\n", err)
} else if isSecretExist {
logboek.Context(ctx).LogF("Cleaning up secret %q ...\n", secret)
if err := kube.Client.CoreV1().Secrets(namespace).Delete(ctx, secret, v1.DeleteOptions{}); err != nil {
logboek.Context(ctx).Warn().LogF("WARNING: secret cleaning up failed: %s\n", err)
}
}
}
Expand Down Expand Up @@ -480,6 +541,45 @@ func createNamespace(ctx context.Context, namespace string) error {
return nil
}

func createDockerRegistrySecret(ctx context.Context, name, namespace string, ref reference.Named, dockerAuthConf imgtypes.DockerAuthConfig) error {
secret := &corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Data: map[string][]byte{},
Type: corev1.SecretTypeDockerConfigJson,
}

var authJson dockerAuthJson
switch {
case dockerAuthConf.IdentityToken != "":
authJson.IdentityToken = dockerAuthConf.IdentityToken
case dockerAuthConf.Username != "" && dockerAuthConf.Password != "":
authJson.Auth = base64.StdEncoding.EncodeToString([]byte(dockerAuthConf.Username + ":" + dockerAuthConf.Password))
default:
panic("unexpected dockerAuthConf")
}

dockerConfJson := &dockerConfigJson{
Auths: map[string]dockerAuthJson{
ref.Name(): authJson,
},
}

dockerConf, err := json.Marshal(dockerConfJson)
if err != nil {
return fmt.Errorf("unable to marshal docker config json: %w", err)
}

secret.Data[corev1.DockerConfigJsonKey] = dockerConf

logboek.Context(ctx).LogF("Creating secret %q in namespace %q ...\n", name, namespace)
kube.Client.CoreV1().Secrets(namespace).Create(ctx, secret, v1.CreateOptions{})

return nil
}

func isNamespaceExist(ctx context.Context, namespace string) (bool, error) {
if matchedNamespaces, err := kube.Client.CoreV1().Namespaces().List(ctx, v1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", namespace).String(),
Expand All @@ -503,3 +603,74 @@ func isPodExist(ctx context.Context, pod string, namespace string) (bool, error)

return false, nil
}

func isSecretExist(ctx context.Context, secret string, namespace string) (bool, error) {
if matchedSecrets, err := kube.Client.CoreV1().Secrets(namespace).List(ctx, v1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", secret).String(),
}); err != nil {
return false, fmt.Errorf("unable to list secrets: %w", err)
} else if len(matchedSecrets.Items) > 0 {
return true, nil
}

return false, nil
}

// Might return empty DockerAuthConfig.
func getDockerConfigCredentials(ref string) (*reference.Named, imgtypes.DockerAuthConfig, error) {
namedRef, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to parse docker config registry reference %q: %w", ref, err)
}

dockerAuthConf, err := config2.GetCredentialsForRef(&imgtypes.SystemContext{AuthFilePath: *commonCmdData.DockerConfig}, namedRef)
if err != nil {
return nil, imgtypes.DockerAuthConfig{}, fmt.Errorf("unable to get docker registry creds for ref %q: %w", ref, err)
}

return &namedRef, dockerAuthConf, nil
}

func addImagePullSecret(secret string, overrides map[string]interface{}) (map[string]interface{}, error) {
if secret == "" {
panic("secret name can't be empty")
}

newImagePullSecret := map[string]interface{}{"name": secret}
newImagePullSecrets := []interface{}{newImagePullSecret}
newSpec := map[string]interface{}{"imagePullSecrets": newImagePullSecrets}
newOverrides := map[string]interface{}{"spec": newSpec}

if len(overrides) == 0 {
return newOverrides, nil
}

if _, ok := overrides["spec"]; !ok {
overrides["spec"] = newSpec
return overrides, nil
}

overridesSpec, ok := overrides["spec"].(map[string]interface{})
if !ok {
return nil, fmt.Errorf("unexpected pod spec overrides format: %+v", overrides)
}

if len(overridesSpec) == 0 {
overrides["spec"] = newSpec
return overrides, nil
}

_, ok = overridesSpec["imagePullSecrets"]
if !ok {
overrides["spec"].(map[string]interface{})["imagePullSecrets"] = newImagePullSecrets
return overrides, nil
}

_, ok = overridesSpec["imagePullSecrets"].([]interface{})
if !ok {
return nil, fmt.Errorf("unexpected imagePullSecrets overrides format: %+v", overrides)
}

overrides["spec"].(map[string]interface{})["imagePullSecrets"] = append(overrides["spec"].(map[string]interface{})["imagePullSecrets"].([]interface{}), newImagePullSecret)
return overrides, nil
}

0 comments on commit d94104f

Please sign in to comment.