diff --git a/Makefile b/Makefile index d5ff655b82..afd8a27647 100644 --- a/Makefile +++ b/Makefile @@ -317,6 +317,32 @@ debug-image-push: --platform=$(PLATFORMS) \ --push ./hack/debugpod +# Build the importer binary +.PHONY: importer-build +importer-build: + $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/importer cmd/importer/main.go + +.PHONY: importer-image-build +importer-image-build: + $(IMAGE_BUILD_CMD) -t $(STAGING_IMAGE_REGISTRY)/importer:$(GIT_TAG) \ + --platform=$(PLATFORMS) \ + --build-arg BASE_IMAGE=$(BASE_IMAGE) \ + --build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \ + --build-arg CGO_ENABLED=$(CGO_ENABLED) \ + $(PUSH) \ + -f ./cmd/importer/Dockerfile ./ + +# Developers don't need to build this image, as it will be available as gcr.io/k8s-staging-kueue/importer +.PHONY: importer-image-push +importer-image-push: PUSH=--push +importer-image-push: importer-image-build + +# Build a docker local gcr.io/k8s-staging-kueue/importer image +.PHONY: importer-image +importer-image: PLATFORMS=linux/amd64 +importer-image: PUSH=--load +importer-image: importer-image-build + PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) GOLANGCI_LINT = $(PROJECT_DIR)/bin/golangci-lint .PHONY: golangci-lint diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 3becf672ab..0494daa3bc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -22,6 +22,15 @@ steps: - GIT_TAG=$_GIT_TAG - EXTRA_TAG=$_PULL_BASE_REF - DOCKER_BUILDX_CMD=/buildx-entrypoint + - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20221214-1b4dd4d69a' + entrypoint: make + args: + - importer-image-push + env: + - IMAGE_REGISTRY=gcr.io/$PROJECT_ID + - GIT_TAG=$_GIT_TAG + - EXTRA_TAG=$_PULL_BASE_REF + - DOCKER_BUILDX_CMD=/buildx-entrypoint substitutions: # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and # can be used as a substitution diff --git a/cmd/importer/Dockerfile b/cmd/importer/Dockerfile new file mode 100644 index 0000000000..19f38b7549 --- /dev/null +++ b/cmd/importer/Dockerfile @@ -0,0 +1,28 @@ +ARG BUILDER_IMAGE +ARG BASE_IMAGE +# Build the manager binary +FROM --platform=${BUILDPLATFORM} ${BUILDER_IMAGE} as builder + +ARG CGO_ENABLED +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY . . + +# Build +RUN make importer-build GO_BUILD_ENV='CGO_ENABLED=${CGO_ENABLED} GOOS=linux GOARCH=${TARGETARCH}' + +FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE} +WORKDIR / +COPY --from=builder /workspace/bin/importer . +USER 65532:65532 + +ENTRYPOINT ["/importer"] diff --git a/cmd/importer/README.md b/cmd/importer/README.md index 4e984fc0bc..0b218021dd 100644 --- a/cmd/importer/README.md +++ b/cmd/importer/README.md @@ -6,13 +6,13 @@ A tool able to import existing pods into kueue. The importer should run in a cluster having the Kueue CRDs defined and in which the `kueue-controller-manager` is not running or has the `pod` integration framework disabled. Check Kueue's [installation guide](https://kueue.sigs.k8s.io/docs/installation/) and [Run Plain Pods](https://kueue.sigs.k8s.io/docs/tasks/run_plain_pods/#before-you-begin) for details. -For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects. +For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects. ## Build From kueue source root run: ```bash -go build -C cmd/importer/ -o $(pwd)/bin/importer +make importer-build ``` @@ -97,3 +97,38 @@ After which, if `--dry-run=false` was specified, for each selected Pod the impor Will import all the pods in namespace `ns1` or `ns2` having the label `src.lbl` set to `src-val` in LocalQueue `user-queue` regardless of their priorityClassName and those with `src.lbl==src-val2` ,`src2.lbl==src2-val` and `priorityClassName==p-class`in `user-queue2`. + +#### Run in cluster + +`cmd/importer/run-in-cluster` provides the necessary kustomize manifests needed to run the importer from within the cluster, In order to use them you should: + +1. Update the used image + +A minimal image containing the importer can be built by + +```bash +make importer-image +``` + +Make the created image accessible by your cluster. + +Note: Importer images will be available in `gcr.io/k8s-staging-kueue/importer` soon. + +And run +```bash +(cd cmd/importer/run-in-cluster && kustomize edit set image importer=) +``` + +2. Updated the importer args in `cmd/importer/run-in-cluster/importer.yaml` +3. Update the mapping configuration in `cmd/importer/run-in-cluster/mapping.yaml` +4. Deploy the configuration: + +```bash + kubectl apply -k cmd/importer/run-in-cluster/ +``` + +And check the logs + +```yaml +kubectl -n kueue-importer logs kueue-importer -f +``` diff --git a/cmd/importer/run-in-cluster/deps.yaml b/cmd/importer/run-in-cluster/deps.yaml new file mode 100644 index 0000000000..f8fe0ff230 --- /dev/null +++ b/cmd/importer/run-in-cluster/deps.yaml @@ -0,0 +1,107 @@ + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kueue-importer +rules: + - verbs: + - get + - list + - patch + - update + apiGroups: + - '' + resources: + - pods + - verbs: + - get + apiGroups: + - '' + resources: + - pods/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues + - verbs: + - get + apiGroups: + - kueue.x-k8s.io + resources: + - clusterqueues/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - localqueues + - verbs: + - get + apiGroups: + - kueue.x-k8s.io + resources: + - localqueues/status + - verbs: + - get + - list + apiGroups: + - kueue.x-k8s.io + resources: + - resourceflavors + - verbs: + - create + - get + - list + - patch + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads + - verbs: + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads/finalizers + - verbs: + - get + - patch + - update + apiGroups: + - kueue.x-k8s.io + resources: + - workloads/status + - verbs: + - get + - list + apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kueue-importer +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kueue-importer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kueue-importer +subjects: +- kind: ServiceAccount + name: kueue-importer +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kueue-importer diff --git a/cmd/importer/run-in-cluster/importer.yaml b/cmd/importer/run-in-cluster/importer.yaml new file mode 100644 index 0000000000..651abcdb8f --- /dev/null +++ b/cmd/importer/run-in-cluster/importer.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kueue-importer +spec: + containers: + - name: importer + image: importer + imagePullPolicy: IfNotPresent + args: + - import + - -n=ns1,ns2 + - --queuemapping-file=/mapping.yaml + - --dry-run=false + - -v + volumeMounts: + - name: config + mountPath: /mapping.yaml + subPath: mapping.yaml + restartPolicy: Never + volumes: + - name: config + configMap: + name: importer-config + serviceAccountName: kueue-importer diff --git a/cmd/importer/run-in-cluster/kustomization.yaml b/cmd/importer/run-in-cluster/kustomization.yaml new file mode 100644 index 0000000000..8199fb6f9e --- /dev/null +++ b/cmd/importer/run-in-cluster/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- importer.yaml +- deps.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - mapping.yaml + name: importer-config + +images: +- name: importer + newName: gcr.io/k8s-staging-kueue/importer + +namespace: kueue-importer diff --git a/cmd/importer/run-in-cluster/mapping.yaml b/cmd/importer/run-in-cluster/mapping.yaml new file mode 100644 index 0000000000..804770807d --- /dev/null +++ b/cmd/importer/run-in-cluster/mapping.yaml @@ -0,0 +1,5 @@ +- match: + labels: + src.lbl: src-val + toLocalQueue: user-queue +- skip: true