From 5b5ec8ae353ba5a79f7de3bee8efb841a043722a Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Fri, 22 Mar 2024 10:33:30 +0200 Subject: [PATCH 1/4] [importer] Run in cluster --- Makefile | 26 +++++ cloudbuild.yaml | 9 ++ cmd/importer/Dockerfile | 28 +++++ cmd/importer/README.md | 39 ++++++- cmd/importer/run-in-cluster/deps.yaml | 107 ++++++++++++++++++ cmd/importer/run-in-cluster/importer.yaml | 25 ++++ .../run-in-cluster/kustomization.yaml | 20 ++++ cmd/importer/run-in-cluster/mapping.yaml | 5 + 8 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 cmd/importer/Dockerfile create mode 100644 cmd/importer/run-in-cluster/deps.yaml create mode 100644 cmd/importer/run-in-cluster/importer.yaml create mode 100644 cmd/importer/run-in-cluster/kustomization.yaml create mode 100644 cmd/importer/run-in-cluster/mapping.yaml diff --git a/Makefile b/Makefile index 54b998f4fb..c20dfeab6b 100644 --- a/Makefile +++ b/Makefile @@ -317,6 +317,32 @@ debug-image-push: --platform=$(PLATFORMS) \ --push ./hack/debugpod +# Build the importer binary +.PHONY: importer-build +importer-build: + $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/importer cmd/importer/main.go + +.PHONY: importer-image-build +importer-image-build: + $(IMAGE_BUILD_CMD) -t $(STAGING_IMAGE_REGISTRY)/importer:$(GIT_TAG) \ + --platform=$(PLATFORMS) \ + --build-arg BASE_IMAGE=$(BASE_IMAGE) \ + --build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \ + --build-arg CGO_ENABLED=$(CGO_ENABLED) \ + $(PUSH) \ + -f ./cmd/importer/Dockerfile ./ + +# Developers don't need to build this image, as it will be available as gcr.io/k8s-staging-kueue/importer +.PHONY: importer-image-push +importer-image-push: PUSH=--push +importer-image-push: importer-image-build + +# Build a docker local gcr.io/k8s-staging-kueue/importer image +.PHONY: importer-image +importer-image: PLATFORMS=linux/amd64 +importer-image: PUSH=--load +importer-image: importer-image-build + PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) GOLANGCI_LINT = $(PROJECT_DIR)/bin/golangci-lint .PHONY: golangci-lint diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 3becf672ab..0494daa3bc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,6 +22,15 @@ steps: - GIT_TAG=$_GIT_TAG - EXTRA_TAG=$_PULL_BASE_REF - DOCKER_BUILDX_CMD=/buildx-entrypoint + - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20221214-1b4dd4d69a' + entrypoint: make + args: + - importer-image-push + env: + - IMAGE_REGISTRY=gcr.io/$PROJECT_ID + - GIT_TAG=$_GIT_TAG + - EXTRA_TAG=$_PULL_BASE_REF + - DOCKER_BUILDX_CMD=/buildx-entrypoint substitutions: # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and # can be used as a substitution diff --git a/cmd/importer/Dockerfile b/cmd/importer/Dockerfile new file mode 100644 index 0000000000..19f38b7549 --- /dev/null +++ b/cmd/importer/Dockerfile @@ -0,0 +1,28 @@ +ARG BUILDER_IMAGE +ARG BASE_IMAGE +# Build the manager binary +FROM --platform=${BUILDPLATFORM} ${BUILDER_IMAGE} as builder + +ARG CGO_ENABLED +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY . . + +# Build +RUN make importer-build GO_BUILD_ENV='CGO_ENABLED=${CGO_ENABLED} GOOS=linux GOARCH=${TARGETARCH}' + +FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE} +WORKDIR / +COPY --from=builder /workspace/bin/importer . +USER 65532:65532 + +ENTRYPOINT ["/importer"] diff --git a/cmd/importer/README.md b/cmd/importer/README.md index 4e984fc0bc..0b218021dd 100644 --- a/cmd/importer/README.md +++ b/cmd/importer/README.md @@ -6,13 +6,13 @@ A tool able to import existing pods into kueue. The importer should run in a cluster having the Kueue CRDs defined and in which the `kueue-controller-manager` is not running or has the `pod` integration framework disabled. Check Kueue's [installation guide](https://kueue.sigs.k8s.io/docs/installation/) and [Run Plain Pods](https://kueue.sigs.k8s.io/docs/tasks/run_plain_pods/#before-you-begin) for details. -For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects. +For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects. ## Build From kueue source root run: ```bash -go build -C cmd/importer/ -o $(pwd)/bin/importer +make importer-build ``` @@ -97,3 +97,38 @@ After which, if `--dry-run=false` was specified, for each selected Pod the impor Will import all the pods in namespace `ns1` or `ns2` having the label `src.lbl` set to `src-val` in LocalQueue `user-queue` regardless of their priorityClassName and those with `src.lbl==src-val2` ,`src2.lbl==src2-val` and `priorityClassName==p-class`in `user-queue2`. + +#### Run in cluster + +`cmd/importer/run-in-cluster` provides the necessary kustomize manifests needed to run the importer from within the cluster, In order to use them you should: + +1. Update the used image + +A minimal image containing the importer can be built by + +```bash +make importer-image +``` + +Make the created image accessible by your cluster. + +Note: Importer images will be available in `gcr.io/k8s-staging-kueue/importer` soon. + +And run +```bash +(cd cmd/importer/run-in-cluster && kustomize edit set image importer=) +``` + +2. Updated the importer args in `cmd/importer/run-in-cluster/importer.yaml` +3. Update the mapping configuration in `cmd/importer/run-in-cluster/mapping.yaml` +4. Deploy the configuration: + +```bash + kubectl apply -k cmd/importer/run-in-cluster/ +``` + +And check the logs + +```yaml +kubectl -n kueue-importer logs kueue-importer -f +``` diff --git a/cmd/importer/run-in-cluster/deps.yaml b/cmd/importer/run-in-cluster/deps.yaml new file mode 100644 index 0000000000..f8fe0ff230 --- /dev/null +++ b/cmd/importer/run-in-cluster/deps.yaml @@ -0,0 +1,107 @@ + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kueue-importer +rules: + - verbs: + - get + - list + - patch + - update + apiGroups: + - '' + resources: + - pods + - verbs: + - get + apiGroups: + - '' + resources: + - pods/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + - verbs: + - get + apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + - verbs: + - get + apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + - verbs: + - create + - get + - list + - patch + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads + - verbs: + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads/finalizers + - verbs: + - get + - patch + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + - verbs: + - get + - list + apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kueue-importer +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kueue-importer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-importer +subjects: +- kind: ServiceAccount + name: kueue-importer +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kueue-importer diff --git a/cmd/importer/run-in-cluster/importer.yaml b/cmd/importer/run-in-cluster/importer.yaml new file mode 100644 index 0000000000..651abcdb8f --- /dev/null +++ b/cmd/importer/run-in-cluster/importer.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kueue-importer +spec: + containers: + - name: importer + image: importer + imagePullPolicy: IfNotPresent + args: + - import + - -n=ns1,ns2 + - --queuemapping-file=/mapping.yaml + - --dry-run=false + - -v + volumeMounts: + - name: config + mountPath: /mapping.yaml + subPath: mapping.yaml + restartPolicy: Never + volumes: + - name: config + configMap: + name: importer-config + serviceAccountName: kueue-importer diff --git a/cmd/importer/run-in-cluster/kustomization.yaml b/cmd/importer/run-in-cluster/kustomization.yaml new file mode 100644 index 0000000000..8199fb6f9e --- /dev/null +++ b/cmd/importer/run-in-cluster/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- importer.yaml +- deps.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - mapping.yaml + name: importer-config + +images: +- name: importer + newName: gcr.io/k8s-staging-kueue/importer + +namespace: kueue-importer diff --git a/cmd/importer/run-in-cluster/mapping.yaml b/cmd/importer/run-in-cluster/mapping.yaml new file mode 100644 index 0000000000..804770807d --- /dev/null +++ b/cmd/importer/run-in-cluster/mapping.yaml @@ -0,0 +1,5 @@ +- match: + labels: + src.lbl: src-val + toLocalQueue: user-queue +- skip: true From eaaf72cb5e1f9dbeb47d5d4a0568ad6a95ccce64 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Fri, 22 Mar 2024 15:55:04 +0200 Subject: [PATCH 2/4] [importer-image] Add a branch specific moving tag --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c20dfeab6b..e48533ae69 100644 --- a/Makefile +++ b/Makefile @@ -324,7 +324,9 @@ importer-build: .PHONY: importer-image-build importer-image-build: - $(IMAGE_BUILD_CMD) -t $(STAGING_IMAGE_REGISTRY)/importer:$(GIT_TAG) \ + $(IMAGE_BUILD_CMD) \ + -t $(STAGING_IMAGE_REGISTRY)/importer:$(GIT_TAG) \ + -t $(STAGING_IMAGE_REGISTRY)/importer:$(RELEASE_BRANCH)-latest \ --platform=$(PLATFORMS) \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \ From 9a16858b54b31310c83bc904de6cf5974da605f5 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Fri, 22 Mar 2024 17:47:08 +0200 Subject: [PATCH 3/4] Review Remarks --- Makefile | 1 - cloudbuild.yaml | 2 +- cmd/importer/README.md | 8 +++++--- cmd/importer/run-in-cluster/importer.yaml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e48533ae69..aedc99b938 100644 --- a/Makefile +++ b/Makefile @@ -334,7 +334,6 @@ importer-image-build: $(PUSH) \ -f ./cmd/importer/Dockerfile ./ -# Developers don't need to build this image, as it will be available as gcr.io/k8s-staging-kueue/importer .PHONY: importer-image-push importer-image-push: PUSH=--push importer-image-push: importer-image-build diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 0494daa3bc..7ef615975f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: args: - importer-image-push env: - - IMAGE_REGISTRY=gcr.io/$PROJECT_ID + - STAGING_IMAGE_REGISTRY=gcr.io/$PROJECT_ID - GIT_TAG=$_GIT_TAG - EXTRA_TAG=$_PULL_BASE_REF - DOCKER_BUILDX_CMD=/buildx-entrypoint diff --git a/cmd/importer/README.md b/cmd/importer/README.md index 0b218021dd..c202bb3be3 100644 --- a/cmd/importer/README.md +++ b/cmd/importer/README.md @@ -100,7 +100,9 @@ After which, if `--dry-run=false` was specified, for each selected Pod the impor #### Run in cluster -`cmd/importer/run-in-cluster` provides the necessary kustomize manifests needed to run the importer from within the cluster, In order to use them you should: +`cmd/importer/run-in-cluster` provides the necessary kustomize manifests needed to run the importer from within the cluster. + +In order to use the manifests, you should: 1. Update the used image @@ -119,7 +121,7 @@ And run (cd cmd/importer/run-in-cluster && kustomize edit set image importer=) ``` -2. Updated the importer args in `cmd/importer/run-in-cluster/importer.yaml` +2. Update the importer args in `cmd/importer/run-in-cluster/importer.yaml` 3. Update the mapping configuration in `cmd/importer/run-in-cluster/mapping.yaml` 4. Deploy the configuration: @@ -127,7 +129,7 @@ And run kubectl apply -k cmd/importer/run-in-cluster/ ``` -And check the logs +And check the logs ```yaml kubectl -n kueue-importer logs kueue-importer -f diff --git a/cmd/importer/run-in-cluster/importer.yaml b/cmd/importer/run-in-cluster/importer.yaml index 651abcdb8f..06a7aed6f0 100644 --- a/cmd/importer/run-in-cluster/importer.yaml +++ b/cmd/importer/run-in-cluster/importer.yaml @@ -12,7 +12,7 @@ spec: - -n=ns1,ns2 - --queuemapping-file=/mapping.yaml - --dry-run=false - - -v + - -vv volumeMounts: - name: config mountPath: /mapping.yaml From 418336e5985bb92a93789abaa8d0066338626ab6 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Fri, 22 Mar 2024 19:53:37 +0200 Subject: [PATCH 4/4] Review Remarks --- Makefile | 2 +- cmd/importer/README.md | 3 +++ cmd/importer/run-in-cluster/importer.yaml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index aedc99b938..8c0ccbd1b0 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ DOCKER_BUILDX_CMD ?= docker buildx IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build IMAGE_BUILD_EXTRA_OPTS ?= # TODO(#52): Add kueue to k8s gcr registry -STAGING_IMAGE_REGISTRY := gcr.io/k8s-staging-kueue +STAGING_IMAGE_REGISTRY ?= gcr.io/k8s-staging-kueue IMAGE_REGISTRY ?= $(STAGING_IMAGE_REGISTRY) IMAGE_NAME := kueue IMAGE_REPO ?= $(IMAGE_REGISTRY)/$(IMAGE_NAME) diff --git a/cmd/importer/README.md b/cmd/importer/README.md index c202bb3be3..e811c30df7 100644 --- a/cmd/importer/README.md +++ b/cmd/importer/README.md @@ -122,6 +122,9 @@ And run ``` 2. Update the importer args in `cmd/importer/run-in-cluster/importer.yaml` + +Note: `dry-run` is set to `false` by default. + 3. Update the mapping configuration in `cmd/importer/run-in-cluster/mapping.yaml` 4. Deploy the configuration: diff --git a/cmd/importer/run-in-cluster/importer.yaml b/cmd/importer/run-in-cluster/importer.yaml index 06a7aed6f0..3c83229d99 100644 --- a/cmd/importer/run-in-cluster/importer.yaml +++ b/cmd/importer/run-in-cluster/importer.yaml @@ -12,7 +12,7 @@ spec: - -n=ns1,ns2 - --queuemapping-file=/mapping.yaml - --dry-run=false - - -vv + - -v=2 volumeMounts: - name: config mountPath: /mapping.yaml