open-policy-agent · sozercan · Apr 8, 2024
@@ -94,6 +94,7 @@ spec:
             - --mutating-webhook-configuration-name={{ .Values.mutatingWebhookName }}
             - --external-data-provider-response-cache-ttl={{ .Values.externaldataProviderResponseCacheTTL }}
             - --experimental-enable-k8s-native-validation={{ .Values.enableK8sNativeValidation }}
+            - --experimental-enable-llm-engine={{ .Values.enableLLMEngine }}
             - --vap-enforcement={{ .Values.vapEnforcement }}
             - HELMBUST_ENABLE_TLS_APISERVER_AUTHENTICATION
             - HELMSUBST_METRICS_BACKEND_ARG

@@ -45,6 +45,7 @@ auditEventsInvolvedNamespace: false
 resourceQuota: true
 externaldataProviderResponseCacheTTL: 3m
 enableK8sNativeValidation: false
+enableLLMEngine: false
 vapEnforcement: GATEKEEPER_DEFAULT
 image:
   repository: openpolicyagent/gatekeeper

@@ -53,6 +53,7 @@ var (
 	flagImages       []string
 	flagTempDir      string
 	flagEnableK8sCel bool
+	flagEnableLLM    bool
 )
 
 const (
@@ -74,6 +75,7 @@ func init() {
 	Cmd.Flags().BoolVarP(&flagIncludeTrace, "trace", "t", false, "include a trace for the underlying Constraint Framework evaluation.")
 	Cmd.Flags().BoolVarP(&flagGatherStats, "stats", "", false, "include performance stats returned from the Constraint Framework.")
 	Cmd.Flags().BoolVarP(&flagEnableK8sCel, "experimental-enable-k8s-native-validation", "", false, "PROTOTYPE (not stable): enable the validating admission policy driver")
+	Cmd.Flags().BoolVarP(&flagEnableLLM, "experimental-enable-llm-engine", "", false, "[Experimental] enable the LLM engine driver")
 	Cmd.Flags().StringArrayVarP(&flagImages, flagNameImage, "i", []string{}, "a URL to an OCI image containing policies. Can be specified multiple times.")
 	Cmd.Flags().StringVarP(&flagTempDir, flagNameTempDir, "d", "", fmt.Sprintf("Specifies the temporary directory to download and unpack images to, if using the --%s flag. Optional.", flagNameImage))
 }
@@ -87,7 +89,13 @@ func run(_ *cobra.Command, _ []string) {
 		cmdutils.ErrFatalf("no input data identified")
 	}
 
-	responses, err := test.Test(unstrucs, test.Opts{IncludeTrace: flagIncludeTrace, GatherStats: flagGatherStats, UseK8sCEL: flagEnableK8sCel})
+	responses, err := test.Test(unstrucs,
+		test.Opts{
+			IncludeTrace: flagIncludeTrace,
+			GatherStats:  flagGatherStats,
+			UseK8sCEL:    flagEnableK8sCel,
+			UseLLM:       flagEnableLLM,
+		})
 	if err != nil {
 		cmdutils.ErrFatalf("auditing objects: %v", err)
 	}

@@ -38,6 +38,7 @@ var (
 	verbose          bool
 	includeTrace     bool
 	flagEnableK8sCel bool
+	flagEnableLLM    bool
 )
 
 func init() {
@@ -49,6 +50,8 @@ func init() {
 		`include a trace for the underlying constraint framework evaluation`)
 	Cmd.Flags().BoolVarP(&flagEnableK8sCel, "experimental-enable-k8s-native-validation", "", false,
 		`PROTOTYPE (not stable): enable the validating admission policy driver`)
+	Cmd.Flags().BoolVarP(&flagEnableLLM, "experimental-enable-llm-engine", "", false,
+		`[Experimental] enable the LLM engine driver`)
 }
 
 // Cmd is the gator verify subcommand.
@@ -112,7 +115,7 @@ func runE(cmd *cobra.Command, args []string) error {
 func runSuites(ctx context.Context, fileSystem fs.FS, suites []*verify.Suite, filter verify.Filter) error {
 	isFailure := false
 
-	runner, err := verify.NewRunner(fileSystem, gator.NewOPAClient, verify.IncludeTrace(includeTrace), verify.UseK8sCEL(flagEnableK8sCel))
+	runner, err := verify.NewRunner(fileSystem, gator.NewOPAClient, verify.IncludeTrace(includeTrace), verify.UseK8sCEL(flagEnableK8sCel), verify.UseLLM(flagEnableLLM))
 	if err != nil {
 		return err
 	}

@@ -67,6 +67,21 @@ spec:
               name: healthz
               protocol: TCP
           env:
+            - name: OPENAI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-api-key
+            - name: OPENAI_DEPLOYMENT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-deployment-name
+            - name: OPENAI_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-endpoint
             # used by Gatekeeper
             - name: POD_NAMESPACE
               valueFrom:
@@ -152,6 +167,21 @@ spec:
             - /manager
           image: openpolicyagent/gatekeeper:v3.16.0-beta.2
           env:
+            - name: OPENAI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-api-key
+            - name: OPENAI_DEPLOYMENT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-deployment-name
+            - name: OPENAI_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: openai-secret
+                  key: openai-endpoint
             # used by Gatekeeper
             - name: POD_NAMESPACE
               valueFrom:
@@ -234,3 +264,13 @@ spec:
       scopeName: PriorityClass
       values:
       - system-cluster-critical
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openai-secret
+  namespace: system
+data:
+  openai-api-key: ""
+  openai-deployment-name: ""
+  openai-endpoint: ""
@@ -0,0 +1,47 @@
+> [!WARNING]
+> This is a demo of a prototype-stage feature and is subject to change. Feedback is welcome!
+
+> [!NOTE]
+> LLM engine can be used in addition to Rego and CEL/K8s Native Validation drivers. It is not a replacement for Rego or CEL.
+>
+> Depending on your provider, LLM engine may have additional costs. Please refer to your provider's pricing details for more information.
+
+## Pre-requisites
+
+- Supports [OpenAI](https://platform.openai.com), [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service), and any other LLM inference engine that supports the OpenAI API, such as [AIKit](https://sozercan.github.io/aikit/) or others. With open weights models, depending on the model, results may not be optimal.
+- For OpenAI and Azure OpenAI, you need GPT 3.5 or GPT 4 (recommended) with a minimum of `1106` or `0125` and later versions.
+
+## Gatekeeper
+- Requires [building Gatekeeper from source](https://open-policy-agent.github.io/gatekeeper/website/docs/install#deploying-head-using-make) and deploying to a Kubernetes cluster.
+- Set `--experimental-enable-llm-engine` in Gatekeeper deployments.
+- Depending on the endpoint, you might need to update validating webhook configuration timeout value.
+- Create or edit the secret called `gatekeeper-openai-secret` in Gatekeeper's namespace (`gatekeeper-system` by default)
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gatekeeper-openai-secret
+  namespace: gatekeeper-system
+data:
+  openai-api-key: # base64 encoded openai api key
+  openai-deployment-name: # base64 encoded openai model or deployment name
+  openai-endpoint: # base64 encoded openai or openai api compatible endpoint. Defaults to https://api.openai.com/v1 if not set.
+```
+
+## Gator
+- Requires building Gator from source with `make gator`.
+- Set the following environment variables:
+```shell
+export OPENAI_API_KEY= # Your OpenAI API key.
+export OPENAI_DEPLOYMENT_NAME= # The deployment or model name used for OpenAI.
+export OPENAI_ENDPOINT= # The endpoint for OpenAI service. Defaults to https://api.openai.com/v1. Set this Azure OpenAI Service or OpenAI API compatible endpoint, if needed.
+```
+- Run Gator with the `--experimental-enable-llm-engine` flag.
+
+```shell
+$ cat example.yaml | bin/gator test --experimental-enable-llm-engine -f demo/llm
+v1/Pod nginx: ["repo-is-dockerio"] Message: "Pod's container is using an image from a disallowed registry: docker.io"
+```
+
+## Demo
+[![asciicast](https://asciinema.org/a/QTBEBp8l0vE5wgD9yGgxE3mnV.svg)](https://asciinema.org/a/QTBEBp8l0vE5wgD9yGgxE3mnV)
@@ -0,0 +1,13 @@
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: K8sAllowedRepos
+metadata:
+  name: repo-is-dockerio
+spec:
+  enforcementAction: deny
+  match:
+    kinds:
+      - apiGroups: [""]
+        kinds: ["Pod"]
+  parameters:
+    repos:
+      - "docker.io/"
@@ -0,0 +1,24 @@
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: k8sallowedrepos
+spec:
+  crd:
+    spec:
+      names:
+        kind: K8sAllowedRepos
+      validation:
+        # Schema for the `parameters` field
+        openAPIV3Schema:
+          type: object
+          properties:
+            repos:
+              type: array
+              items:
+                type: string
+  targets:
+    - target: admission.k8s.gatekeeper.sh
+      code:
+        - engine: LLM
+          source:
+            prompt: "do not allow images from registries from the parameters list"
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Pre-requisites:
+# Kubernetes cluster that has Gatekepeer with LLM engine installed
+# Gator in path
+# Kubectl AI (https://github.com/sozercan/kubectl-ai) in path
+
+. ../../third_party/demo-magic/demo-magic.sh
+
+clear
+
+echo "🎬 This is a demo for Gatekeeper with LLM engine."
+
+pe "kubectl get pods -n gatekeeper-system"
+
+echo "🔐 Deploying a policy that requires no privileged containers"
+
+pe "bat privileged-template.yaml"
+
+pe "kubectl apply -f privileged-template.yaml"
+
+pe "bat privileged-constraint.yaml"
+
+pe "kubectl apply -f privileged-constraint.yaml"
+
+pe "kubectl ai 'create a pod that is privileged'"
+
+echo "👉 As expected, this pod got denied because it is privileged"
+
+pe "kubectl ai 'create a pod that is not privileged'"
+
+echo "👉 As expected, this pod did not get denied because it is not privileged"
+
+echo "✅ Deploying a policy that requires images to be from allowed registries"
+
+pe "bat allowedrepos-template.yaml"
+
+pe "kubectl apply -f allowedrepos-template.yaml"
+
+pe "bat allowedrepos-constraint.yaml"
+
+pe "kubectl apply -f allowedrepos-constraint.yaml"
+
+pe "kubectl ai 'create a pod with docker.io/library/nginx:latest image'"
+
+echo "👉 As expected, this pod got denied because it is from a disapproved registry"
+
+pe "kubectl ai 'create a pod with registry.k8s.io/pause:3.8 image'"
+
+echo "👉 As expected, this pod did not get denied because it is not from a disapproved registry"
+
+echo "🐊 Using Gator for shift left validation"
+
+pe "kubectl ai 'create a pod with docker.io/nginx:latest image that is privileged' --raw | gator test --experimental-enable-llm-engine -f ."
+
+echo "👉 As expected, we were able to validate this Kubernetes manifest with Gator for shift left validation"
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: openai-secret
+  namespace: gatekeeper-system
+data:
+  openai-api-key: # base64 encoded openai api key
+  openai-deployment-name: # base64 encoded openai model or deployment name
+  openai-endpoint: # base64 encoded openai or openai api compatible endpoint
@@ -0,0 +1,14 @@
+apiVersion: constraints.gatekeeper.sh/v1beta1
+kind: K8sPSPPrivilegedContainer
+metadata:
+  name: psp-privileged-container
+spec:
+  enforcementAction: deny
+  match:
+    kinds:
+      - apiGroups: [""]
+        kinds: ["Pod"]
+    excludedNamespaces: ["kube-system"]
+  parameters:
+    exemptImages:
+      - "docker.io/library/python:latest"
@@ -0,0 +1,23 @@
+apiVersion: templates.gatekeeper.sh/v1
+kind: ConstraintTemplate
+metadata:
+  name: k8spspprivilegedcontainer
+spec:
+  crd:
+    spec:
+      names:
+        kind: K8sPSPPrivilegedContainer
+      validation:
+        openAPIV3Schema:
+          type: object
+          properties:
+            exemptImages:
+              type: array
+              items:
+                type: string
+  targets:
+    - target: admission.k8s.gatekeeper.sh
+      code:
+        - engine: LLM
+          source:
+            prompt: "do not allow privileged containers except for images from the exemptImages list"
@@ -4860,7 +4860,7 @@ spec:
           value: manager
         - name: OTEL_RESOURCE_ATTRIBUTES
           value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE),k8s.container.name=$(CONTAINER_NAME)
-        image: openpolicyagent/gatekeeper:v3.16.0-beta.2
+        image: sozercan/gatekeeper:llm
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -4979,7 +4979,7 @@ spec:
           value: manager
         - name: OTEL_RESOURCE_ATTRIBUTES
           value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE),k8s.container.name=$(CONTAINER_NAME)
-        image: openpolicyagent/gatekeeper:v3.16.0-beta.2
+        image: sozercan/gatekeeper:llm
         imagePullPolicy: Always
         livenessProbe:
           httpGet:

@@ -62,7 +62,7 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go v1.47.9 // indirect
+	github.com/aws/aws-sdk-go v1.50.10 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
@@ -78,7 +78,7 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -119,10 +119,13 @@ require (
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/sashabaranov/go-openai v1.20.4 // indirect
+	github.com/sethvargo/go-retry v0.2.4 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
+	github.com/walles/env v0.0.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yashtewari/glob-intersection v0.2.0 // indirect
@@ -141,6 +144,7 @@ require (
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/api v0.169.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
@@ -155,3 +159,5 @@ require (
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
+
+replace github.com/open-policy-agent/frameworks/constraint => ../frameworks/constraint