From 9c143b7882eda9d0598027878e1809ee4fde5135 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Fri, 27 Dec 2024 23:37:21 +0000 Subject: [PATCH 1/8] adding vap testing Signed-off-by: Jaydip Gabani --- .github/workflows/workflow.yaml | 4 ++-- Makefile | 10 +++++----- test/bats/test.bats | 13 ++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index e426d299e..584b332d3 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -66,7 +66,7 @@ jobs: strategy: matrix: gatekeeper: [ "3.17.1", "3.18.1" ] - engine: [ "cel", "rego" ] + engine: [ "cel", "rego", "vap" ] name: "Integration test on Gatekeeper ${{ matrix.gatekeeper }} for ${{ matrix.engine }} policies" steps: - name: Harden Runner @@ -86,7 +86,7 @@ jobs: - name: Run integration test run: | - make test-integration + make test-integration POLICY_ENGINE=${{ matrix.engine }} - name: Save logs run: | diff --git a/Makefile b/Makefile index 064e6ec94..4af96b328 100755 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ docker := docker #You can build with podman by doing: make docker=podman KIND_VERSION ?= 0.23.0 # note: k8s version pinned since KIND image availability lags k8s releases -KUBERNETES_VERSION ?= 1.30.0 +KUBERNETES_VERSION ?= 1.32.0 KUSTOMIZE_VERSION ?= 4.5.5 -GATEKEEPER_VERSION ?= 3.16.3 +GATEKEEPER_VERSION ?= 3.18.1 BATS_VERSION ?= 1.8.2 -GATOR_VERSION ?= 3.17.0 +GATOR_VERSION ?= 3.18.1 GOMPLATE_VERSION ?= 3.11.6 POLICY_ENGINE ?= rego @@ -36,9 +36,9 @@ deploy: ifeq ($(POLICY_ENGINE), rego) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=false else ifeq ($(POLICY_ENGINE), cel) -ifneq ($(GATEKEEPER_VERSION), 3.15.1) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true -endif +else ifeq ($(POLICY_ENGINE), vap) + helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true --set defaultCreateVAPForTemplates=true --set defaultCreateVAPBindingForConstraints=true endif uninstall: diff --git a/test/bats/test.bats b/test/bats/test.bats index f45441166..e96ef8b4c 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -85,9 +85,15 @@ setup() { if [ -d "$policy" ]; then local policy_group=$(basename "$(dirname "$policy")") local template_name=$(basename "$policy") + vapb_exists=false + deny_substr="denied the request" echo "running integration test against policy group: $policy_group, constraint template: $template_name" # apply template wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -k $policy" + if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$template_name" + vapb_exists=true + fi local kind=$(yq e .metadata.name "$policy"/template.yaml) for sample in "$policy"/samples/*; do echo "testing sample constraint: $(basename "$sample")" @@ -96,6 +102,11 @@ setup() { local name=$(yq e .metadata.name "$sample"/constraint.yaml) wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" + if [ vapb_exists == true ]; then + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicyBinding gatekeeper-$name" + deny_substr="ValidatingAdmissionPolicy" + fi + for inventory in "$sample"/example_inventory*.yaml; do if [[ -e "$inventory" ]]; then run kubectl apply -f "$inventory" @@ -123,7 +134,7 @@ setup() { echo "Applying ${disallowed} with contents:" cat ${disallowed} run kubectl apply -f "$disallowed" - assert_match_either 'denied the request' 'no matches for kind' "${output}" + assert_match_either "$deny_substr" 'no matches for kind' "${output}" assert_failure # delete resource run kubectl delete --ignore-not-found -f "$disallowed" From 0af8b18a54170b1dd4fe206b226d020a7bdc4b4a Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Sat, 28 Dec 2024 07:18:13 +0000 Subject: [PATCH 2/8] fixing vap tests Signed-off-by: Jaydip Gabani --- Makefile | 2 +- test/bats/test.bats | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 4af96b328..722c4d330 100755 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ docker := docker #You can build with podman by doing: make docker=podman KIND_VERSION ?= 0.23.0 # note: k8s version pinned since KIND image availability lags k8s releases -KUBERNETES_VERSION ?= 1.32.0 +KUBERNETES_VERSION ?= 1.30.0 KUSTOMIZE_VERSION ?= 4.5.5 GATEKEEPER_VERSION ?= 3.18.1 BATS_VERSION ?= 1.8.2 diff --git a/test/bats/test.bats b/test/bats/test.bats index e96ef8b4c..b099cc7a4 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -85,16 +85,15 @@ setup() { if [ -d "$policy" ]; then local policy_group=$(basename "$(dirname "$policy")") local template_name=$(basename "$policy") - vapb_exists=false deny_substr="denied the request" echo "running integration test against policy group: $policy_group, constraint template: $template_name" # apply template wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -k $policy" + local kind=$(yq e .metadata.name "$policy"/template.yaml) if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$template_name" - vapb_exists=true + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$kind" + deny_substr="ValidatingAdmissionPolicy" fi - local kind=$(yq e .metadata.name "$policy"/template.yaml) for sample in "$policy"/samples/*; do echo "testing sample constraint: $(basename "$sample")" # apply constraint @@ -102,9 +101,8 @@ setup() { local name=$(yq e .metadata.name "$sample"/constraint.yaml) wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" - if [ vapb_exists == true ]; then + if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicyBinding gatekeeper-$name" - deny_substr="ValidatingAdmissionPolicy" fi for inventory in "$sample"/example_inventory*.yaml; do From cd32b82dcb496e4a006ee257f8c72d53a5702224 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Tue, 31 Dec 2024 22:26:53 +0000 Subject: [PATCH 3/8] waiting before creating constraints Signed-off-by: Jaydip Gabani --- test/bats/test.bats | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/bats/test.bats b/test/bats/test.bats index b099cc7a4..96c102c27 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -2,7 +2,7 @@ load helpers -TESTS_DIR=library +TESTS_DIR=/mount/d/go/src/github.com/open-policy-agent/gatekeeper-library/library BATS_TESTS_DIR=test/bats WAIT_TIME=300 SLEEP_TIME=5 @@ -89,16 +89,17 @@ setup() { echo "running integration test against policy group: $policy_group, constraint template: $template_name" # apply template wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -k $policy" - local kind=$(yq e .metadata.name "$policy"/template.yaml) + local kind=$(cat "$policy"/template.yaml | yq e .metadata.name) if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$kind" + sleep 30 deny_substr="ValidatingAdmissionPolicy" fi for sample in "$policy"/samples/*; do echo "testing sample constraint: $(basename "$sample")" # apply constraint wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -f ${sample}/constraint.yaml" - local name=$(yq e .metadata.name "$sample"/constraint.yaml) + local name=$(cat "$sample"/constraint.yaml | yq e .metadata.name) wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then From b3eeb00520807cdf67d66cbb065788085ecf0919 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Tue, 31 Dec 2024 22:41:15 +0000 Subject: [PATCH 4/8] fixing test path Signed-off-by: Jaydip Gabani --- test/bats/test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/test.bats b/test/bats/test.bats index 96c102c27..c73645dcb 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -2,7 +2,7 @@ load helpers -TESTS_DIR=/mount/d/go/src/github.com/open-policy-agent/gatekeeper-library/library +TESTS_DIR=library BATS_TESTS_DIR=test/bats WAIT_TIME=300 SLEEP_TIME=5 From 1ce5632f2becf271a730615dba6d4620ef9b18f5 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Mon, 6 Jan 2025 22:28:58 +0000 Subject: [PATCH 5/8] separating vap tests for more clarity Signed-off-by: Jaydip Gabani --- .github/workflows/workflow.yaml | 41 ++++++++++++++++++++++++++++++++- Makefile | 4 +++- test/bats/test.bats | 4 ++-- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 584b332d3..ff00faf4e 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -66,7 +66,7 @@ jobs: strategy: matrix: gatekeeper: [ "3.17.1", "3.18.1" ] - engine: [ "cel", "rego", "vap" ] + engine: [ "cel", "rego"] name: "Integration test on Gatekeeper ${{ matrix.gatekeeper }} for ${{ matrix.engine }} policies" steps: - name: Harden Runner @@ -100,6 +100,45 @@ jobs: name: logs-int-test-${{ matrix.gatekeeper }}-${{ matrix.engine }} path: | logs-*.json + build_test_VAP: + needs: generate + runs-on: ubuntu-latest + strategy: + matrix: + gatekeeper: [ "3.17.1", "3.18.1" ] + name: "Integration test on Gatekeeper ${{ matrix.gatekeeper }} with VAP" + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - name: Check out code into the Go module directory + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Bootstrap integration test + run: | + mkdir -p $GITHUB_WORKSPACE/bin + echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH + make integration-bootstrap + make deploy GATEKEEPER_VERSION=${{ matrix.gatekeeper }} ENABLE_VAP=true + + - name: Run integration test + run: | + make test-integration ENABLE_VAP=true + + - name: Save logs + run: | + kubectl logs -n gatekeeper-system -l control-plane=controller-manager --tail=-1 > logs-controller.json + kubectl logs -n gatekeeper-system -l control-plane=audit-controller --tail=-1 > logs-audit.json + + - name: Upload artifacts + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + if: ${{ always() }} + with: + name: logs-int-test-${{ matrix.gatekeeper }}-with-vap + path: | + logs-*.json require_suites: runs-on: ubuntu-latest name: "Require a suite.yaml file alongside every template.yaml" diff --git a/Makefile b/Makefile index 722c4d330..88fd4e96a 100755 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ BATS_VERSION ?= 1.8.2 GATOR_VERSION ?= 3.18.1 GOMPLATE_VERSION ?= 3.11.6 POLICY_ENGINE ?= rego +ENABLE_VAP ?= false REPO_ROOT := $(shell git rev-parse --show-toplevel) WEBSITE_SCRIPT_DIR := $(REPO_ROOT)/scripts/website @@ -33,11 +34,12 @@ integration-bootstrap: deploy: helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts +# If the policy engine is rego, enableK8sNativeValidation should be set to false because K8sNativeValidation engine holds more priority than Rego engine. Otherwise Rego engine will not get evaluated for CT containing K8sNativeValidation engine. ifeq ($(POLICY_ENGINE), rego) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=false else ifeq ($(POLICY_ENGINE), cel) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true -else ifeq ($(POLICY_ENGINE), vap) +else ifeq ($(ENABLE_VAP), true) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true --set defaultCreateVAPForTemplates=true --set defaultCreateVAPBindingForConstraints=true endif diff --git a/test/bats/test.bats b/test/bats/test.bats index c73645dcb..d654ad0f5 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -90,7 +90,7 @@ setup() { # apply template wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -k $policy" local kind=$(cat "$policy"/template.yaml | yq e .metadata.name) - if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then + if [ "$ENABLE_VAP" == "true" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$kind" sleep 30 deny_substr="ValidatingAdmissionPolicy" @@ -102,7 +102,7 @@ setup() { local name=$(cat "$sample"/constraint.yaml | yq e .metadata.name) wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" - if [ "$POLICY_ENGINE" == "vap" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then + if [ "$ENABLE_VAP" == "true" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicyBinding gatekeeper-$name" fi From 4dd94aff6c3a395e8a3fe188a54faf97f3a689a1 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Mon, 6 Jan 2025 22:50:57 +0000 Subject: [PATCH 6/8] fixing vap deployment Signed-off-by: Jaydip Gabani --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 88fd4e96a..a651d9ba0 100755 --- a/Makefile +++ b/Makefile @@ -35,12 +35,12 @@ integration-bootstrap: deploy: helm repo add gatekeeper https://open-policy-agent.github.io/gatekeeper/charts # If the policy engine is rego, enableK8sNativeValidation should be set to false because K8sNativeValidation engine holds more priority than Rego engine. Otherwise Rego engine will not get evaluated for CT containing K8sNativeValidation engine. -ifeq ($(POLICY_ENGINE), rego) - helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=false +ifeq ($(ENABLE_VAP), true) + helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true --set defaultCreateVAPForTemplates=true --set defaultCreateVAPBindingForConstraints=true else ifeq ($(POLICY_ENGINE), cel) helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true -else ifeq ($(ENABLE_VAP), true) - helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=true --set defaultCreateVAPForTemplates=true --set defaultCreateVAPBindingForConstraints=true +else ifeq ($(POLICY_ENGINE), rego) + helm install -n gatekeeper-system gatekeeper gatekeeper/gatekeeper --create-namespace --version $(GATEKEEPER_VERSION) --set enableK8sNativeValidation=false endif uninstall: From 89f849ecc695496183012e76f9343411f0e9bf56 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Tue, 7 Jan 2025 18:48:16 +0000 Subject: [PATCH 7/8] debugging VAPB Signed-off-by: Jaydip Gabani --- test/bats/test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/test.bats b/test/bats/test.bats index d654ad0f5..ea4952d69 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -103,7 +103,7 @@ setup() { wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" if [ "$ENABLE_VAP" == "true" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicyBinding gatekeeper-$name" + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicyBinding gatekeeper-$name -o yaml" fi for inventory in "$sample"/example_inventory*.yaml; do From 5681b2c51d3c79ab91a837c42d86f492333b4794 Mon Sep 17 00:00:00 2001 From: Jaydip Gabani Date: Tue, 7 Jan 2025 19:10:56 +0000 Subject: [PATCH 8/8] debugging VAP Signed-off-by: Jaydip Gabani --- test/bats/test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bats/test.bats b/test/bats/test.bats index ea4952d69..335742870 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -91,7 +91,7 @@ setup() { wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -k $policy" local kind=$(cat "$policy"/template.yaml | yq e .metadata.name) if [ "$ENABLE_VAP" == "true" ] && grep -q "engine: K8sNativeValidation" "$policy"/template.yaml; then - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$kind" + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl get ValidatingAdmissionPolicy gatekeeper-$kind -o yaml" sleep 30 deny_substr="ValidatingAdmissionPolicy" fi