[importer] Run in cluster

Makefile

@@ -317,6 +317,34 @@ debug-image-push:
 		--platform=$(PLATFORMS) \
 		--push ./hack/debugpod
 
+# Build the importer binary
+.PHONY: importer-build
+importer-build:
+	$(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/importer cmd/importer/main.go
+
+.PHONY: importer-image-build
+importer-image-build:
+	$(IMAGE_BUILD_CMD) \
+		-t $(STAGING_IMAGE_REGISTRY)/importer:$(GIT_TAG) \
+		-t $(STAGING_IMAGE_REGISTRY)/importer:$(RELEASE_BRANCH)-latest \
+		--platform=$(PLATFORMS) \
+		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
+		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
+		--build-arg CGO_ENABLED=$(CGO_ENABLED) \
+		$(PUSH) \
+		-f ./cmd/importer/Dockerfile ./
+
+# Developers don't need to build this image, as it will be available as gcr.io/k8s-staging-kueue/importer
+.PHONY: importer-image-push
+importer-image-push: PUSH=--push
+importer-image-push: importer-image-build
+
+# Build a docker local gcr.io/k8s-staging-kueue/importer image
+.PHONY: importer-image
+importer-image: PLATFORMS=linux/amd64
+importer-image: PUSH=--load
+importer-image: importer-image-build
+
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 GOLANGCI_LINT = $(PROJECT_DIR)/bin/golangci-lint
 .PHONY: golangci-lint

cloudbuild.yaml

@@ -22,6 +22,15 @@ steps:
       - GIT_TAG=$_GIT_TAG
       - EXTRA_TAG=$_PULL_BASE_REF
       - DOCKER_BUILDX_CMD=/buildx-entrypoint
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20221214-1b4dd4d69a'
+    entrypoint: make
+    args:
+      - importer-image-push
+    env:
+      - IMAGE_REGISTRY=gcr.io/$PROJECT_ID
+      - GIT_TAG=$_GIT_TAG
+      - EXTRA_TAG=$_PULL_BASE_REF
+      - DOCKER_BUILDX_CMD=/buildx-entrypoint
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
   # can be used as a substitution

cmd/importer/Dockerfile

@@ -0,0 +1,28 @@
+ARG BUILDER_IMAGE
+ARG BASE_IMAGE
+# Build the manager binary
+FROM --platform=${BUILDPLATFORM} ${BUILDER_IMAGE} as builder
+
+ARG CGO_ENABLED
+ARG TARGETARCH
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY . .
+
+# Build
+RUN make importer-build GO_BUILD_ENV='CGO_ENABLED=${CGO_ENABLED} GOOS=linux GOARCH=${TARGETARCH}'
+
+FROM --platform=${BUILDPLATFORM} ${BASE_IMAGE}
+WORKDIR /
+COPY --from=builder /workspace/bin/importer .
+USER 65532:65532
+
+ENTRYPOINT ["/importer"]

cmd/importer/README.md

@@ -6,13 +6,13 @@ A tool able to import existing pods into kueue.
 
 The importer should run in a cluster having the Kueue CRDs defined and in which the `kueue-controller-manager` is not running or has the `pod` integration framework disabled. Check Kueue's [installation guide](https://kueue.sigs.k8s.io/docs/installation/) and [Run Plain Pods](https://kueue.sigs.k8s.io/docs/tasks/run_plain_pods/#before-you-begin) for details.
 
-For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects. 
+For an import to succeed, all the involved Kueue objects (LocalQueues, ClusterQueues and ResourceFlavors) need to be created in the cluster, the check stage of the importer will check this and enumerate the missing objects.
 
 ## Build
 
 From kueue source root run:
  ```bash
-go build  -C cmd/importer/ -o $(pwd)/bin/importer
+make importer-build
 
  ```
 
@@ -97,3 +97,38 @@ After which, if `--dry-run=false` was specified, for each selected Pod the impor
 
  Will import all the pods in namespace `ns1` or `ns2` having the label `src.lbl` set to `src-val` in LocalQueue `user-queue` regardless of their priorityClassName and those with `src.lbl==src-val2` ,`src2.lbl==src2-val` and `priorityClassName==p-class`in `user-queue2`.
 
+
+#### Run in cluster
+
+`cmd/importer/run-in-cluster` provides the necessary kustomize manifests needed to run the importer from within the cluster, In order to use them you should:
+
+1. Update the used image
+
+A minimal image containing the importer can be built by
+
+```bash
+make importer-image
+```
+
+Make the created image accessible by your cluster.
+
+Note: Importer images will be available in `gcr.io/k8s-staging-kueue/importer` soon.
+
+And run
+```bash
+(cd cmd/importer/run-in-cluster && kustomize edit set image importer=<image:tag>)
+```
+
+2. Updated the importer args in `cmd/importer/run-in-cluster/importer.yaml`
+3. Update the mapping configuration in `cmd/importer/run-in-cluster/mapping.yaml`
+4. Deploy the configuration:
+
+```bash
+ kubectl apply -k cmd/importer/run-in-cluster/
+```
+
+And check  the logs
+
+```yaml
+kubectl -n kueue-importer logs kueue-importer -f
+```

cmd/importer/run-in-cluster/deps.yaml

@@ -0,0 +1,107 @@
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kueue-importer
+rules:
+  - verbs:
+      - get
+      - list
+      - patch
+      - update
+    apiGroups:
+      - ''
+    resources:
+      - pods
+  - verbs:
+      - get
+    apiGroups:
+      - ''
+    resources:
+      - pods/status
+  - verbs:
+      - get
+      - list
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - clusterqueues
+  - verbs:
+      - get
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - clusterqueues/status
+  - verbs:
+      - get
+      - list
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - localqueues
+  - verbs:
+      - get
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - localqueues/status
+  - verbs:
+      - get
+      - list
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - resourceflavors
+  - verbs:
+      - create
+      - get
+      - list
+      - patch
+      - update
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - workloads
+  - verbs:
+      - update
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - workloads/finalizers
+  - verbs:
+      - get
+      - patch
+      - update
+    apiGroups:
+      - kueue.x-k8s.io
+    resources:
+      - workloads/status
+  - verbs:
+      - get
+      - list
+    apiGroups:
+      - scheduling.k8s.io
+    resources:
+      - priorityclasses
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kueue-importer
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kueue-importer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kueue-importer
+subjects:
+- kind: ServiceAccount
+  name: kueue-importer
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kueue-importer 

cmd/importer/run-in-cluster/importer.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kueue-importer
+spec:
+  containers:
+  - name: importer
+    image: importer
+    imagePullPolicy: IfNotPresent
+    args:
+    - import
+    - -n=ns1,ns2
+    - --queuemapping-file=/mapping.yaml
+    - --dry-run=false
+    - -v
+    volumeMounts:
+    - name: config
+      mountPath: /mapping.yaml
+      subPath: mapping.yaml
+  restartPolicy: Never
+  volumes:
+    - name: config
+      configMap:
+        name: importer-config
+  serviceAccountName: kueue-importer

cmd/importer/run-in-cluster/kustomization.yaml

@@ -0,0 +1,20 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- importer.yaml
+- deps.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configMapGenerator:
+- files:
+  - mapping.yaml
+  name: importer-config
+
+images:
+- name: importer
+  newName: gcr.io/k8s-staging-kueue/importer
+
+namespace: kueue-importer

cmd/importer/run-in-cluster/mapping.yaml

@@ -0,0 +1,5 @@
+- match:
+    labels:
+      src.lbl: src-val
+  toLocalQueue: user-queue
+- skip: true