Merge pull request #6950 from inteon/improve_codegen

Improve Makefile generate and verify targets (make them parallelizable)

hack/k8s-codegen.sh

@@ -18,15 +18,13 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-go=$1
-
-clientgen=$2
-deepcopygen=$3
-informergen=$4
-listergen=$5
-defaultergen=$6
-conversiongen=$7
-openapigen=$8
+clientgen=$1
+deepcopygen=$2
+informergen=$3
+listergen=$4
+defaultergen=$5
+conversiongen=$6
+openapigen=$7
 
 # If the envvar "VERIFY_ONLY" is set, we only check if everything's up to date
 # and don't actually generate anything
@@ -124,18 +122,8 @@ clean() {
   find "$path" -name "$name" -delete
 }
 
-mkcp() {
-  src="$1"
-  dst="$2"
-  mkdir -p "$(dirname "$dst")"
-  cp "$src" "$dst"
-}
-
-# Export mkcp for use in sub-shells
-export -f mkcp
-
 gen-openapi-acme() {
-  clean pkg/acme/webhook/openapi '*.go'
+  clean pkg/acme/webhook/openapi 'zz_generated.openapi.go'
   echo "+++ ${VERB} ACME openapi..." >&2
   mkdir -p hack/openapi_reports
   "$openapigen" \

klone.yaml

@@ -10,35 +10,35 @@ targets:
     - folder_name: boilerplate
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: 04f424fa90aa8ca570278cf0c07b18dea607b542
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/boilerplate
     - folder_name: generate-verify
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: 04f424fa90aa8ca570278cf0c07b18dea607b542
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/generate-verify
     - folder_name: go
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: fa9c9274d1d852de501461b9442f7206aaf74007
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/go
     - folder_name: help
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: 04f424fa90aa8ca570278cf0c07b18dea607b542
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/help
     - folder_name: klone
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: 04f424fa90aa8ca570278cf0c07b18dea607b542
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/klone
     - folder_name: repository-base
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: e9363accaaee20a995bbf8f1c9cba2ea77da8935
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/repository-base
     - folder_name: tools
       repo_url: https://github.com/cert-manager/makefile-modules.git
       repo_ref: main
-      repo_hash: 04f424fa90aa8ca570278cf0c07b18dea607b542
+      repo_hash: ad721163bbe8d8d755d54c88a2b2475aeb7c79de
       repo_path: modules/tools

make/02_mod.mk

@@ -36,20 +36,29 @@ include make/e2e-setup.mk
 include make/scan.mk
 include make/ko.mk
 
+.PHONY: go-workspace
+go-workspace: export GOWORK?=$(abspath go.work)
+## Create a go.work file in the repository root (or GOWORK)
+##
+## @category Development
+go-workspace: | $(NEEDS_GO)
+	@rm -f $(GOWORK)
+	$(GO) work init
+	$(GO) work use . ./cmd/acmesolver ./cmd/cainjector ./cmd/controller ./cmd/startupapicheck ./cmd/webhook ./test/integration ./test/e2e
 
 .PHONY: tidy
 ## Run "go mod tidy" on each module in this repo
 ##
 ## @category Development
-tidy:
-	go mod tidy
-	cd cmd/acmesolver && go mod tidy
-	cd cmd/cainjector && go mod tidy
-	cd cmd/controller && go mod tidy
-	cd cmd/startupapicheck && go mod tidy
-	cd cmd/webhook && go mod tidy
-	cd test/integration && go mod tidy
-	cd test/e2e && go mod tidy
+tidy: | $(NEEDS_GO)
+	$(GO) mod tidy
+	cd cmd/acmesolver && $(GO) mod tidy
+	cd cmd/cainjector && $(GO) mod tidy
+	cd cmd/controller && $(GO) mod tidy
+	cd cmd/startupapicheck && $(GO) mod tidy
+	cd cmd/webhook && $(GO) mod tidy
+	cd test/integration && $(GO) mod tidy
+	cd test/e2e && $(GO) mod tidy
 
 .PHONY: update-base-images
 update-base-images: | $(NEEDS_CRANE)

make/_shared/generate-verify/00_mod.mk

@@ -13,5 +13,6 @@
 # limitations under the License.
 
 shared_generate_targets ?=
+shared_generate_targets_dirty ?=
 shared_verify_targets ?=
 shared_verify_targets_dirty ?=

make/_shared/generate-verify/02_mod.mk

@@ -16,18 +16,24 @@
 ## Generate all generate targets.
 ## @category [shared] Generate/ Verify
 generate: $$(shared_generate_targets)
+	@echo "The following targets cannot be run simultaniously with each other or other generate scripts:"
+	$(foreach TARGET,$(shared_generate_targets_dirty), $(MAKE) $(TARGET))
 
 verify_script := $(dir $(lastword $(MAKEFILE_LIST)))/util/verify.sh
 
 # Run the supplied make target argument in a temporary workspace and diff the results.
 verify-%: FORCE
-	$(verify_script) $(MAKE) -s $*
+	+$(verify_script) $(MAKE) $*
 
 verify_generated_targets = $(shared_generate_targets:%=verify-%)
+verify_generated_targets_dirty = $(shared_generate_targets_dirty:%=verify-%)
+
+verify_targets = $(sort $(verify_generated_targets) $(shared_verify_targets))
+verify_targets_dirty = $(sort $(verify_generated_targets_dirty) $(shared_verify_targets_dirty))
 
 .PHONY: verify
 ## Verify code and generate targets.
 ## @category [shared] Generate/ Verify
-verify: $$(verify_generated_targets) $$(shared_verify_targets)
+verify: $$(verify_targets)
 	@echo "The following targets create temporary files in the current directory, that is why they have to be run last:"
-	$(MAKE) noop $(shared_verify_targets_dirty)
+	$(foreach TARGET,$(verify_targets_dirty), $(MAKE) $(TARGET))

make/_shared/generate-verify/util/verify.sh

@@ -45,6 +45,7 @@ cleanup() {
 trap "cleanup" EXIT SIGINT
 
 cp -a "${projectdir}/." "${tmp}"
+rm -rf "${tmp}/_bin" # clear all cached files
 pushd "${tmp}" >/dev/null
 
 "$@"

make/_shared/go/01_mod.mk

@@ -72,7 +72,7 @@ shared_generate_targets += generate-golangci-lint-config
 .PHONY: verify-golangci-lint
 ## Verify all Go modules using golangci-lint
 ## @category [shared] Generate/ Verify
-verify-golangci-lint: | $(NEEDS_GOLANGCI-LINT) $(NEEDS_YQ) $(bin_dir)/scratch
+verify-golangci-lint: | $(NEEDS_GO) $(NEEDS_GOLANGCI-LINT) $(NEEDS_YQ) $(bin_dir)/scratch
 	@find . -name go.mod -not \( -path "./$(bin_dir)/*" -or -path "./make/_shared/*" \) -printf '%h\n' \
 		| while read d; do \
 				echo "Running '$(bin_dir)/tools/golangci-lint run --go $(VENDORED_GO_VERSION) -c $(CURDIR)/$(golangci_lint_config)' in directory '$${d}'"; \