Added the chainsaw tests for kyverno 1.10 release (#116)

* Added the chainsaw tests for kyverno 1.10 release

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* Updated the k8s version matrix

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* Updated failure action for the policies

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* UPdated the pss disallow proc mount test case

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* UPdated the pss disallow proc mount test case

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* Added the disallow-privilege-escalation policy changes

* Migrate the kuttl test for workload securty policy to chainsaw

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

* Bump the chart version

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

---------

Signed-off-by: nsathyaseelan <sathyaseelan@nirmata.com>

.chainsaw-config.yaml

@@ -0,0 +1,17 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Configuration
+metadata:
+  creationTimestamp: null
+  name: configuration
+spec:
+  parallel: 1
+  timeouts:
+    apply: 1m30s
+    assert: 1m30s
+    cleanup: 2m30s
+    delete: 1m30s
+    error: 1m30s
+    exec: 1m30s
+  fullName: true
+  forceTerminationGracePeriod: 5s
+  delayBeforeCleanup: 3s

.github/workflows/chainsaw-e2e.yaml

@@ -0,0 +1,46 @@
+name: ChainSaw Test
+on:
+  push:
+    branches:
+      - 'release-chart-1.10'
+
+  pull_request:
+    branches:
+      - 'release-chart-1.10'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-e2etest:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false    
+      matrix:
+        k8s-version: [v1.29.2, v1.28.7, v1.27.11, v1.26.14, v1.25.16, v1.24.12, v1.23.17]
+        # For n4k-versions 1.10
+        n4k-chart-version: [3.0.18]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Prepare environment
+        run: K8S_VERSION=${{ matrix.k8s-version }} make kind-create-cluster
+
+      - name: Install kyverno
+        run: N4K_VERSION=${{ matrix.n4k-chart-version }} make kind-deploy-kyverno
+
+      - name: Check Kyverno status
+        run: make wait-for-kyverno
+
+      - name: Install Chainsaw
+        uses: kyverno/action-install-chainsaw@v0.1.4 
+
+      - name: Verify Chainsaw Installation
+        run: chainsaw version
+
+      - name: Test with Chainsaw
+        run: make test-chainsaw

Makefile

@@ -1,18 +1,16 @@
 .DEFAULT_GOAL: build-all
 
 K8S_VERSION          ?= $(shell kubectl version --short | grep -i server | cut -d" " -f3 | cut -c2-)
-KIND_IMAGE           ?= kindest/node:v1.27.1
+KIND_IMAGE           ?= kindest/node:$(K8S_VERSION)
 KIND_NAME            ?= kind
 USE_CONFIG           ?= standard
 
 TOOLS_DIR                          := $(PWD)/.tools
 KIND                               := $(TOOLS_DIR)/kind
-KIND_VERSION                       := v0.19.0
+KIND_VERSION                       := v0.22.0
 HELM                               := $(TOOLS_DIR)/helm
 HELM_VERSION                       := v3.10.1
-KUTTL                              := $(TOOLS_DIR)/kubectl-kuttl
-KUTTL_VERSION                      := v0.0.0-20230108220859-ef8d83c89156
-TOOLS                              := $(KIND) $(HELM) $(KUTTL)
+TOOLS                              := $(KIND) $(HELM)
 
 $(KIND):
 	@echo Install kind... >&2
@@ -22,10 +20,6 @@ $(HELM):
 	@echo Install helm... >&2
 	@GOBIN=$(TOOLS_DIR) go install helm.sh/helm/v3/cmd/helm@$(HELM_VERSION)
 
-$(KUTTL):
-	@echo Install kuttl... >&2
-	@GOBIN=$(TOOLS_DIR) go install github.com/kyverno/kuttl/cmd/kubectl-kuttl@$(KUTTL_VERSION)
-
 .PHONY: install-tools
 install-tools: $(TOOLS)
 
@@ -34,20 +28,20 @@ clean-tools:
 	@echo Clean tools... >&2
 	@rm -rf $(TOOLS_DIR)
 
-###############
-# KUTTL TESTS #
-###############
+##################
+# CHAINSAW TESTS #
+##################
 
-.PHONY: test-kuttl
-test-kuttl: $(KUTTL) ## Run kuttl tests
-	@echo Running kuttl tests... >&2
-	@$(KUTTL) test --config kuttl-test.yaml
+.PHONY: test-chainsaw
+test-chainsaw:  
+	@echo Running chainsaw tests... >&2
+	@chainsaw test --config .chainsaw-config.yaml
 
 ## Create kind cluster
 .PHONY: kind-create-cluster
 kind-create-cluster: $(KIND) 
 	@echo Create kind cluster... >&2
-	@$(KIND) create cluster --name $(KIND_NAME) 
+	@$(KIND) create cluster --name $(KIND_NAME) --image $(KIND_IMAGE)
 
 ## Delete kind cluster
 .PHONY: kind-delete-cluster
@@ -59,21 +53,15 @@ kind-delete-cluster: $(KIND)
 .PHONY: kind-deploy-kyverno
 kind-deploy-kyverno: $(HELM) 
 	@echo Install kyverno chart... >&2
-	@echo $(N4K_LICENSE_KEY) >&2
-
-    ### Adding temporary  installation command for the kyverno n4k 1.10
-    ##  git clone -b release-chart-1.10 https://github.com/nirmata/kyverno-charts.git
-    ## @$(HELM) install kyverno ./kyverno-charts/charts/nirmata -n kyverno --create-namespace  --set licenseManager.licenseKey=+7BT76LNHCKLi3vW2mbYP5vYuS+Rm4XaLPu7k6Vgq4/efR3BEJk6Ru+zOFJagN2l0oLyG15qZ2kkXpzqaeEAal6APDLB7s3htLFeJ6mf0hc7/3dupUY13zrdX5svkS5p6BNKVisuXwK5XfF8sJyLn16I/CRdICj9fzktWQWYB5h46xOj5NlMPMj0/m6tCa3hIVJpB9Onkd4KMXlO+PQUbUwk/wxuciQkGwjbXQs+V9w0MuWMODpY0jGN1dgLNETI7mpS6G5DVvHkbAtrJ+gvG15aFFtKjgPInoemqxbhj2wzYue5pNSdHUZYE9b+LLlj
-
 	@$(HELM) repo add nirmata https://nirmata.github.io/kyverno-charts
-	@$(HELM) repo update nirmata
-	@$(HELM) install kyverno --namespace kyverno --create-namespace nirmata/kyverno --set licenseManager.licenseKey=+7BT76LNHCKLi3vW2mbYP5vYuS+Rm4XaLPu7k6Vgq4/efR3BEJk6Ru+zOFJagN2l0oLyG15qZ2kkXpzqaeEAal6APDLB7s3htLFeJ6mf0hc7/3dupUY13zrdX5svkS5p6BNKVisuXwK5XfF8sJyLn16I/CRdICj9fzktWQWYB5h46xOj5NlMPMj0/m6tCa3hIVJpB9Onkd4KMXlO+PQUbUwk/wxuciQkGwjbXQs+V9w0MuWMODpY0jGN1dgLNETI7mpS6G5DVvHkbAtrJ+gvG15aFFtKjgPInoemqxbhj2wzYue5pNSdHUZYE9b+LLlj --devel
+	@$(HELM) repo update
+	@$(HELM) install kyverno nirmata/kyverno -n kyverno --create-namespace --version=$(N4K_VERSION)
 
 ## Check Kyverno status 
 .PHONY: wait-for-kyverno
 wait-for-kyverno: 
 	@echo Check kyverno status to be ready... >&2
-	@kubectl wait --namespace kyverno --for=condition=ready pod --all --timeout=120s
+	@kubectl wait --namespace kyverno --for=condition=ready pod --all --timeout=180s
 
 #####################
 # Kyverno CLI TESTS #