Merge pull request #212 from stefanprodan/bundle-mod-download

Pull all modules before attempting apply
stefanprodan · Oct 8, 2023 · a5f6a97 · a5f6a97
2 parents 36290ec + 0cefd65
commit a5f6a97
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 84 deletions.
diff --git a/cmd/timoni/artifact_push.go b/cmd/timoni/artifact_push.go
@@ -160,7 +160,12 @@ func pushArtifactCmdRun(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	log.Info(fmt.Sprintf("digest: %s", colorizeSubject(digestURL)))
+	digest, err := oci.ParseDigest(digestURL)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("artifact: %s", colorizeSubject(ociURL)))
+	log.Info(fmt.Sprintf("digest: %s", colorizeSubject(digest.DigestStr())))
 
 	return nil
 }
diff --git a/cmd/timoni/bundle_apply.go b/cmd/timoni/bundle_apply.go
@@ -22,6 +22,7 @@ import (
 	"io"
 	"maps"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -99,6 +100,7 @@ func init() {
 }
 
 func runBundleApplyCmd(cmd *cobra.Command, _ []string) error {
+	start := time.Now()
 	files := bundleApplyArgs.files
 	for i, file := range files {
 		if file == "-" {
@@ -174,55 +176,64 @@ func runBundleApplyCmd(cmd *cobra.Command, _ []string) error {
 		}
 	}
 
-	log.Info(fmt.Sprintf("applying %v instance(s)", len(bundle.Instances)))
+	ctxPull, cancel := context.WithTimeout(ctx, rootArgs.timeout)
+	defer cancel()
+
+	for _, instance := range bundle.Instances {
+		spin := StartSpinner(fmt.Sprintf("pulling %s", instance.Module.Repository))
+		pullErr := fetchBundleInstanceModule(ctxPull, instance, tmpDir)
+		spin.Stop()
+		if pullErr != nil {
+			return pullErr
+		}
+	}
 
 	kubeVersion, err := runtime.ServerVersion(kubeconfigArgs)
 	if err != nil {
 		return err
 	}
 
+	if bundleApplyArgs.dryrun || bundleApplyArgs.diff {
+		log.Info(fmt.Sprintf("applying %v instance(s) %s",
+			len(bundle.Instances), colorizeDryRun("(server dry run)")))
+	} else {
+		log.Info(fmt.Sprintf("applying %v instance(s)",
+			len(bundle.Instances)))
+	}
+
 	for _, instance := range bundle.Instances {
-		log.Info(fmt.Sprintf("applying instance %s", instance.Name))
-		if err := applyBundleInstance(logr.NewContext(ctx, log), cuectx, instance, kubeVersion); err != nil {
+		if err := applyBundleInstance(logr.NewContext(ctx, log), cuectx, instance, kubeVersion, tmpDir); err != nil {
 			return err
 		}
 	}
 
+	elapsed := time.Since(start)
 	if bundleApplyArgs.dryrun || bundleApplyArgs.diff {
-		log.Info(fmt.Sprintf("applied %v instance(s) (server dry run)", len(bundle.Instances)))
+		log.Info(fmt.Sprintf("applied successfully %s",
+			colorizeDryRun("(server dry run)")))
 	} else {
-		log.Info("applied successfully")
+		log.Info(fmt.Sprintf("applied successfully in %s", elapsed.Round(time.Second)))
 	}
 
 	return nil
 }
 
-func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engine.BundleInstance, kubeVersion string) error {
-	moduleVersion := instance.Module.Version
-	sourceURL := fmt.Sprintf("%s:%s", instance.Module.Repository, instance.Module.Version)
+func fetchBundleInstanceModule(ctx context.Context, instance *engine.BundleInstance, rootDir string) error {
+	modDir := path.Join(rootDir, instance.Name)
+	if err := os.MkdirAll(modDir, os.ModePerm); err != nil {
+		return err
+	}
 
+	moduleVersion := instance.Module.Version
 	if moduleVersion == apiv1.LatestVersion && instance.Module.Digest != "" {
-		sourceURL = fmt.Sprintf("%s@%s", instance.Module.Repository, instance.Module.Digest)
 		moduleVersion = "@" + instance.Module.Digest
 	}
 
-	log := LoggerBundleInstance(ctx, instance.Bundle, instance.Name)
-	log.Info(fmt.Sprintf("pulling %s", sourceURL))
-
-	tmpDir, err := os.MkdirTemp("", apiv1.FieldManager)
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
-	defer cancel()
-
 	fetcher := engine.NewFetcher(
-		ctxPull,
+		ctx,
 		instance.Module.Repository,
 		moduleVersion,
-		tmpDir,
+		modDir,
 		bundleApplyArgs.creds.String(),
 	)
 	mod, err := fetcher.Fetch()
@@ -235,35 +246,44 @@ func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engi
 			mod.Digest, instance.Module.Version, instance.Module.Digest)
 	}
 
+	instance.Module = *mod
+	return nil
+}
+
+func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance *engine.BundleInstance, kubeVersion string, rootDir string) error {
+	log := LoggerBundleInstance(ctx, instance.Bundle, instance.Name)
+
+	modDir := path.Join(rootDir, instance.Name, "module")
 	builder := engine.NewModuleBuilder(
 		cuectx,
 		instance.Name,
 		instance.Namespace,
-		fetcher.GetModuleRoot(),
+		modDir,
 		bundleApplyArgs.pkg.String(),
 	)
 
 	if err := builder.WriteSchemaFile(); err != nil {
 		return err
 	}
 
-	mod.Name, err = builder.GetModuleName()
+	modName, err := builder.GetModuleName()
 	if err != nil {
 		return err
 	}
+	instance.Module.Name = modName
 
-	log.Info(fmt.Sprintf("using module %s version %s", mod.Name, mod.Version))
-
+	log.Info(fmt.Sprintf("applying module %s version %s",
+		colorizeSubject(instance.Module.Name), colorizeSubject(instance.Module.Version)))
 	err = builder.WriteValuesFileWithDefaults(instance.Values)
 	if err != nil {
 		return err
 	}
 
-	builder.SetVersionInfo(mod.Version, kubeVersion)
+	builder.SetVersionInfo(instance.Module.Version, kubeVersion)
 
 	buildResult, err := builder.Build()
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "failed to build instance", err)
+		return describeErr(modDir, "failed to build instance", err)
 	}
 
 	finalValues, err := builder.GetValues(buildResult)
@@ -299,7 +319,7 @@ func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engi
 		return fmt.Errorf("instance init failed: %w", err)
 	}
 
-	im := runtime.NewInstanceManager(instance.Name, instance.Namespace, finalValues, *mod)
+	im := runtime.NewInstanceManager(instance.Name, instance.Namespace, finalValues, instance.Module)
 
 	if im.Instance.Labels == nil {
 		im.Instance.Labels = make(map[string]string)
@@ -317,18 +337,28 @@ func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engi
 
 	if bundleApplyArgs.dryrun || bundleApplyArgs.diff {
 		if !nsExists {
-			log.Info(colorizeJoin(colorizeNamespaceFromArgs(), ssa.CreatedAction, dryRunServer))
+			log.Info(colorizeJoin(colorizeSubject("Namespace/"+instance.Namespace),
+				ssa.CreatedAction, dryRunServer))
 		}
-		if err := instanceDryRunDiff(logr.NewContext(ctx, log), rm, objects, staleObjects, nsExists, tmpDir, bundleApplyArgs.diff); err != nil {
+		if err := instanceDryRunDiff(
+			logr.NewContext(ctx, log),
+			rm,
+			objects,
+			staleObjects,
+			nsExists,
+			rootDir,
+			bundleApplyArgs.diff,
+		); err != nil {
 			return err
 		}
 
-		log.Info("applied successfully (server dry run)")
+		log.Info(colorizeJoin("applied successfully", colorizeDryRun("(server dry run)")))
 		return nil
 	}
 
 	if !exists {
-		log.Info(fmt.Sprintf("installing %s in namespace %s", instance.Name, instance.Namespace))
+		log.Info(fmt.Sprintf("installing %s in namespace %s",
+			colorizeSubject(instance.Name), colorizeSubject(instance.Namespace)))
 
 		if err := sm.Apply(ctx, &im.Instance, true); err != nil {
 			return fmt.Errorf("instance init failed: %w", err)
@@ -338,7 +368,8 @@ func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engi
 			log.Info(colorizeJoin(colorizeSubject("Namespace/"+instance.Namespace), ssa.CreatedAction))
 		}
 	} else {
-		log.Info(fmt.Sprintf("upgrading %s in namespace %s", instance.Name, instance.Namespace))
+		log.Info(fmt.Sprintf("upgrading %s in namespace %s",
+			colorizeSubject(instance.Name), colorizeSubject(instance.Namespace)))
 	}
 
 	applyOpts := runtime.ApplyOptions(bundleApplyArgs.force, rootArgs.timeout)
@@ -407,7 +438,7 @@ func applyBundleInstance(ctx context.Context, cuectx *cue.Context, instance engi
 	return nil
 }
 
-func bundleInstancesOwnershipConflicts(bundleInstances []engine.BundleInstance) error {
+func bundleInstancesOwnershipConflicts(bundleInstances []*engine.BundleInstance) error {
 	var conflicts []string
 	rm, err := runtime.NewResourceManager(kubeconfigArgs)
 	if err != nil {

diff --git a/cmd/timoni/bundle_apply_test.go b/cmd/timoni/bundle_apply_test.go
@@ -304,7 +304,6 @@ bundle: {
 		output, err := executeCommandWithIn("bundle apply -f - -p main --wait", r)
 		g.Expect(err).ToNot(HaveOccurred())
 		g.Expect(output).To(ContainSubstring(modVer1))
-		g.Expect(output).To(ContainSubstring(modDigestv1))
 	})
 
 	t.Run("creates instance for module version digest", func(t *testing.T) {

diff --git a/cmd/timoni/bundle_build.go b/cmd/timoni/bundle_build.go
@@ -21,9 +21,11 @@ import (
 	"fmt"
 	"maps"
 	"os"
+	"path"
 	"sort"
 	"strings"
 
+	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/cuecontext"
 	"github.com/fluxcd/pkg/ssa"
 	"github.com/spf13/cobra"
@@ -142,12 +144,21 @@ func runBundleBuildCmd(cmd *cobra.Command, _ []string) error {
 
 	var sb strings.Builder
 
+	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
+	defer cancel()
+
+	for _, instance := range bundle.Instances {
+		if err := fetchBundleInstanceModule(ctxPull, instance, tmpDir); err != nil {
+			return err
+		}
+	}
+
 	for i, instance := range bundle.Instances {
 		sb.WriteString("---\n")
 		sb.WriteString(fmt.Sprintf("# Instance: %s\n", instance.Name))
 		sb.WriteString("---\n")
 
-		instance, err := buildBundleInstance(instance)
+		instance, err := buildBundleInstance(ctx, instance, tmpDir)
 		if err != nil {
 			return err
 		}
@@ -163,65 +174,37 @@ func runBundleBuildCmd(cmd *cobra.Command, _ []string) error {
 	return nil
 }
 
-func buildBundleInstance(instance engine.BundleInstance) (string, error) {
-	moduleVersion := instance.Module.Version
-
-	if moduleVersion == apiv1.LatestVersion && instance.Module.Digest != "" {
-		moduleVersion = "@" + instance.Module.Digest
-	}
-
-	tmpDir, err := os.MkdirTemp("", apiv1.FieldManager)
-	if err != nil {
-		return "", err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
-	defer cancel()
+func buildBundleInstance(cuectx *cue.Context, instance *engine.BundleInstance, rootDir string) (string, error) {
+	modDir := path.Join(rootDir, instance.Name, "module")
 
-	fetcher := engine.NewFetcher(
-		ctxPull,
-		instance.Module.Repository,
-		moduleVersion,
-		tmpDir,
-		bundleBuildArgs.creds.String(),
-	)
-	mod, err := fetcher.Fetch()
-	if err != nil {
-		return "", err
-	}
-
-	if instance.Module.Digest != "" && mod.Digest != instance.Module.Digest {
-		return "", fmt.Errorf("the upstream digest %s of version %s doesn't match the specified digest %s",
-			mod.Digest, instance.Module.Version, instance.Module.Digest)
-	}
-
-	cuectx := cuecontext.New()
 	builder := engine.NewModuleBuilder(
 		cuectx,
 		instance.Name,
 		instance.Namespace,
-		fetcher.GetModuleRoot(),
+		modDir,
 		bundleBuildArgs.pkg.String(),
 	)
 
 	if err := builder.WriteSchemaFile(); err != nil {
 		return "", err
 	}
 
-	mod.Name, err = builder.GetModuleName()
+	modName, err := builder.GetModuleName()
 	if err != nil {
 		return "", err
 	}
+	instance.Module.Name = modName
 
 	err = builder.WriteValuesFileWithDefaults(instance.Values)
 	if err != nil {
 		return "", err
 	}
 
+	builder.SetVersionInfo(instance.Module.Version, "")
+
 	buildResult, err := builder.Build()
 	if err != nil {
-		return "", describeErr(fetcher.GetModuleRoot(), "failed to build instance", err)
+		return "", describeErr(modDir, "failed to build instance", err)
 	}
 
 	bundleBuildSets, err := builder.GetApplySets(buildResult)

diff --git a/cmd/timoni/bundle_delete.go b/cmd/timoni/bundle_delete.go
@@ -110,7 +110,7 @@ func deleteBundleByName(ctx context.Context, bundle string) error {
 	for index := len(instances) - 1; index >= 0; index-- {
 		instance := instances[index]
 		log.Info(fmt.Sprintf("deleting instance %s from bundle %s", instance.Name, bundleDelArgs.name))
-		if err := deleteBundleInstance(ctx, engine.BundleInstance{
+		if err := deleteBundleInstance(ctx, &engine.BundleInstance{
 			Bundle:    bundle,
 			Name:      instance.Name,
 			Namespace: instance.Namespace,
@@ -176,7 +176,7 @@ func deleteBundleFromFile(ctx context.Context, cmd *cobra.Command) error {
 	return nil
 }
 
-func deleteBundleInstance(ctx context.Context, instance engine.BundleInstance, wait bool, dryrun bool) error {
+func deleteBundleInstance(ctx context.Context, instance *engine.BundleInstance, wait bool, dryrun bool) error {
 	log := LoggerBundle(ctx, instance.Bundle)
 
 	sm, err := runtime.NewResourceManager(kubeconfigArgs)

diff --git a/cmd/timoni/mod_pull.go b/cmd/timoni/mod_pull.go
@@ -137,19 +137,17 @@ func pullCmdRun(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	spin := StartSpinner("pulling module")
-	defer spin.Stop()
-
 	ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
+	spin := StartSpinner(fmt.Sprintf("pulling %s", ociURL))
 	opts := oci.Options(ctx, pullModArgs.creds.String())
 	err := oci.PullArtifact(ociURL, pullModArgs.output, apiv1.AnyContentType, opts)
+	spin.Stop()
 	if err != nil {
 		return err
 	}
 
-	spin.Stop()
 	log.Info(fmt.Sprintf("extracted: %s", colorizeSubject(pullModArgs.output)))
 
 	return nil

diff --git a/cmd/timoni/mod_push.go b/cmd/timoni/mod_push.go
@@ -201,6 +201,7 @@ func pushModCmdRun(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
+		log.Info(fmt.Sprintf("artifact: %s", colorizeSubject(ociURL)))
 		log.Info(fmt.Sprintf("digest: %s", colorizeSubject(digest.DigestStr())))
 	}