[project/gateway2]: Adds Support for GatewayClass SupportedVersion Co…

…ndition Previously, the gateway2 controller would unconditionally set the GatewayClass Accepted condition to `true`. This PR adds a watch for the Gateway API CRDs that triggers GatewayClass reconciliation. The `ReconcileGatewayClasses()` method was updated to set GatewayClass status conditionas based on the Gateway API spec. Fixes: #10092 Signed-off-by: Daneyon Hansen <[email protected]>
solo-io · Oct 1, 2024 · d234e90 · d234e90
1 parent d4ab5d4
commit d234e90
Show file tree

Hide file tree

Showing 15 changed files with 468 additions and 19 deletions.
diff --git a/changelog/v1.18.0-beta25/issue_10092.yaml b/changelog/v1.18.0-beta25/issue_10092.yaml
@@ -0,0 +1,7 @@
+changelog:
+  - type: FIX
+    issueLink: https://github.com/solo-io/gloo/issues/10092
+    resolvesIssue: true
+    description: >-
+      Adds support for the GatewayClass SupportedVersion status condition and updates the ReconcileGatewayClasses()
+      method to manage status conditions according to the Gateway API specification.
diff --git a/install/helm/gloo/templates/44-rbac.yaml b/install/helm/gloo/templates/44-rbac.yaml
@@ -29,6 +29,11 @@ rules:
   resources:
   - endpointslices
   verbs: ["get", "list", "watch"]
+- apiGroups:
+  - "apiextensions.k8s.io"
+  resources:
+  - customresourcedefinitions
+  verbs: ["get", "list", "watch"]
 - apiGroups:
   - "gateway.solo.io"
   resources:

diff --git a/pkg/schemes/scheme.go b/pkg/schemes/scheme.go
@@ -9,6 +9,7 @@ import (
 	ratelimitv1alpha1 "github.com/solo-io/solo-apis/pkg/api/ratelimit.solo.io/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
 	apiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
@@ -25,6 +26,9 @@ var SchemeBuilder = runtime.SchemeBuilder{
 	corev1.AddToScheme,
 	appsv1.AddToScheme,
 
+	// Kubernetes apiextensions resources
+	apiextv1.AddToScheme,
+
 	// Solo Kubernetes Gateway API resources
 	sologatewayv1alpha1.AddToScheme,
 

diff --git a/projects/gateway2/controller/controller.go b/projects/gateway2/controller/controller.go
@@ -3,8 +3,11 @@ package controller
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -24,6 +27,7 @@ import (
 
 	sologatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1/kube/apis/gateway.solo.io/v1"
 	"github.com/solo-io/gloo/projects/gateway2/api/v1alpha1"
+	"github.com/solo-io/gloo/projects/gateway2/crds"
 	"github.com/solo-io/gloo/projects/gateway2/deployer"
 	"github.com/solo-io/gloo/projects/gateway2/extensions"
 	"github.com/solo-io/gloo/projects/gateway2/query"
@@ -62,13 +66,15 @@ func NewBaseGatewayController(ctx context.Context, cfg GatewayConfig) error {
 	controllerBuilder := &controllerBuilder{
 		cfg: cfg,
 		reconciler: &controllerReconciler{
-			cli:    cfg.Mgr.GetClient(),
-			scheme: cfg.Mgr.GetScheme(),
-			kick:   cfg.Kick,
+			controllerName: cfg.ControllerName,
+			cli:            cfg.Mgr.GetClient(),
+			scheme:         cfg.Mgr.GetScheme(),
+			kick:           cfg.Kick,
 		},
 	}
 
 	return run(ctx,
+		controllerBuilder.watchCRDs,
 		controllerBuilder.watchGwClass,
 		controllerBuilder.watchGw,
 		controllerBuilder.watchHttpRoute,
@@ -244,6 +250,26 @@ func shouldIgnoreStatusChild(gvk schema.GroupVersionKind) bool {
 	return gvk.Kind == "Deployment"
 }
 
+// watchCRDs sets up a controller to watch for changes to specific Gateway API CRDs and triggers
+// GatewayClass reconciliation if generation or annotations change.
+func (c *controllerBuilder) watchCRDs(ctx context.Context) error {
+	return ctrl.NewControllerManagedBy(c.cfg.Mgr).
+		For(&apiextv1.CustomResourceDefinition{}).
+		WithEventFilter(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			// We care about changes to the specific CRDs
+			crd, ok := object.(*apiextv1.CustomResourceDefinition)
+			if !ok {
+				return false
+			}
+			return crds.IsSupported(crd.Name)
+		})).
+		WithEventFilter(predicate.Or(
+			predicate.AnnotationChangedPredicate{},
+			predicate.GenerationChangedPredicate{},
+		)).
+		Complete(reconcile.Func(c.reconciler.ReconcileCRDs))
+}
+
 func (c *controllerBuilder) watchGwClass(_ context.Context) error {
 	return ctrl.NewControllerManagedBy(c.cfg.Mgr).
 		WithEventFilter(predicate.GenerationChangedPredicate{}).
@@ -330,9 +356,10 @@ func (c *controllerBuilder) watchDirectResponses(_ context.Context) error {
 }
 
 type controllerReconciler struct {
-	cli    client.Client
-	scheme *runtime.Scheme
-	kick   func(ctx context.Context)
+	controllerName string
+	cli            client.Client
+	scheme         *runtime.Scheme
+	kick           func(ctx context.Context)
 }
 
 func (r *controllerReconciler) ReconcileHttpListenerOptions(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -408,8 +435,39 @@ func (r *controllerReconciler) ReconcileReferenceGrants(ctx context.Context, req
 	return ctrl.Result{}, nil
 }
 
+func (r *controllerReconciler) ReconcileCRDs(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := log.FromContext(ctx).WithValues("CustomResourceDefinition", req.Name)
+
+	log.Info("reconciling CustomResourceDefinition")
+
+	var gatewayClassList apiv1.GatewayClassList
+	if err := r.cli.List(ctx, &gatewayClassList); err != nil {
+		log.Error(err, "Failed to list GatewayClasses")
+		return ctrl.Result{}, err
+	}
+
+	for _, gatewayClass := range gatewayClassList.Items {
+		if gatewayClass.Spec.ControllerName == apiv1.GatewayController(r.controllerName) {
+			req := ctrl.Request{
+				NamespacedName: client.ObjectKey{
+					Name: gatewayClass.Name,
+				},
+			}
+
+			if _, err := r.ReconcileGatewayClasses(ctx, req); err != nil {
+				log.Error(err, "Failed to reconcile GatewayClass", "name", gatewayClass.Name)
+				return ctrl.Result{}, err
+			}
+		}
+	}
+
+	log.Info("Reconciled CustomResourceDefinition")
+
+	return ctrl.Result{}, nil
+}
+
 func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	log := log.FromContext(ctx).WithValues("gwclass", req.NamespacedName)
+	log := log.FromContext(ctx).WithValues("GatewayClass", req.Name)
 
 	gwclass := &apiv1.GatewayClass{}
 	if err := r.cli.Get(ctx, req.NamespacedName, gwclass); err != nil {
@@ -420,31 +478,74 @@ func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	log.Info("reconciling gateway class")
+	log.Info("Reconciling GatewayClass")
 
-	// mark it as accepted:
+	// Initialize the status conditions. No need to set LastTransitionTime since it's handled by SetStatusCondition.
+	msg := "Gateway API CRDs are a supported version"
 	acceptedCondition := metav1.Condition{
 		Type:               string(apiv1.GatewayClassConditionStatusAccepted),
 		Status:             metav1.ConditionTrue,
 		Reason:             string(apiv1.GatewayClassReasonAccepted),
+		Message:            msg,
 		ObservedGeneration: gwclass.Generation,
-		// no need to set LastTransitionTime, it will be set automatically by SetStatusCondition
 	}
-	meta.SetStatusCondition(&gwclass.Status.Conditions, acceptedCondition)
-
-	// TODO: This should actually check the version of the CRDs in the cluster to be 100% sure
-	supportedVersionCondition := metav1.Condition{
+	supportedCondition := metav1.Condition{
 		Type:               string(apiv1.GatewayClassConditionStatusSupportedVersion),
 		Status:             metav1.ConditionTrue,
-		ObservedGeneration: gwclass.Generation,
 		Reason:             string(apiv1.GatewayClassReasonSupportedVersion),
+		Message:            msg,
+		ObservedGeneration: gwclass.Generation,
+	}
+
+	// Check CRD versions
+	supported, err := r.checkCRDVersions(ctx)
+	if err != nil {
+		log.Error(err, "Failed to check CRD versions")
+		return ctrl.Result{}, err
+	}
+
+	if !supported {
+		// Update the values of status conditions
+		msg = fmt.Sprintf("Unsupported Gateway API CRDs detected. Supported versions are: %s",
+			strings.Join(wellknown.SupportedVersions, ", "))
+		acceptedCondition.Status = metav1.ConditionFalse
+		acceptedCondition.Reason = string(apiv1.GatewayClassReasonUnsupportedVersion)
+		acceptedCondition.Message = msg
+		supportedCondition.Status = metav1.ConditionFalse
+		supportedCondition.Reason = string(apiv1.GatewayClassReasonUnsupportedVersion)
+		supportedCondition.Message = msg
 	}
-	meta.SetStatusCondition(&gwclass.Status.Conditions, supportedVersionCondition)
+
+	// Set the status conditions
+	meta.SetStatusCondition(&gwclass.Status.Conditions, acceptedCondition)
+	meta.SetStatusCondition(&gwclass.Status.Conditions, supportedCondition)
 
 	if err := r.cli.Status().Update(ctx, gwclass); err != nil {
+		log.Error(err, "Failed to update GatewayClass status")
 		return ctrl.Result{}, err
 	}
-	log.Info("updated gateway class status")
+
+	log.Info("Reconciled GatewayClass")
 
 	return ctrl.Result{}, nil
 }
+
+// checkCRDVersions checks that the "gateway.networking.k8s.io/bundle-version" annotation key is set
+// to a supported version for each required Gateway API CRD.
+func (r *controllerReconciler) checkCRDVersions(ctx context.Context) (bool, error) {
+	for _, crdName := range crds.Required {
+		crd := &apiextv1.CustomResourceDefinition{}
+		if err := r.cli.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
+			if errors.IsNotFound(err) {
+				return false, nil
+			}
+			return false, err
+		}
+
+		bundleVersion, exists := crd.Annotations[crds.BundleVersionAnnotation]
+		if !exists || !crds.IsSupportedVersion(bundleVersion) {
+			return false, nil
+		}
+	}
+	return true, nil
+}
diff --git a/projects/gateway2/crds/crds.go b/projects/gateway2/crds/crds.go
@@ -2,7 +2,43 @@ package crds
 
 import (
 	_ "embed"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/solo-io/gloo/projects/gateway2/wellknown"
+)
+
+const (
+	// BundleVersionAnnotation is the annotation key used in the Gateway API CRDs to specify
+	// the installed Gateway API version.
+	// TODO: Use CRD annotation const from sigs.k8s.io/gateway-api/pkg/consts when v1.1.0 is supported.
+	BundleVersionAnnotation = "gateway.networking.k8s.io/bundle-version"
+	// GatewayClass is the name of the GatewayClass CRD.
+	GatewayClass = "gatewayclasses.gateway.networking.k8s.io"
+	// Gateway is the name of the Gateway CRD.
+	Gateway = "gateways.gateway.networking.k8s.io"
+	// HTTPRoute is the name of the HTTPRoute CRD.
+	HTTPRoute = "httproutes.gateway.networking.k8s.io"
+	// ReferenceGrant is the name of the ReferenceGrant CRD.
+	ReferenceGrant = "referencegrants.gateway.networking.k8s.io"
 )
 
 //go:embed gateway-crds.yaml
 var GatewayCrds []byte
+
+// Required is a list of required Gateway API CRDs.
+var Required = []string{GatewayClass, Gateway, HTTPRoute, ReferenceGrant}
+
+// IsSupportedVersion checks if the CRD version is recognized and supported.
+func IsSupportedVersion(version string) bool {
+	supportedVersions := sets.NewString(wellknown.SupportedVersions...)
+	return supportedVersions.Has(version)
+}
+
+// IsSupported checks if the CRD is supported based on the provided name.
+func IsSupported(name string) bool {
+	return name == GatewayClass ||
+		name == Gateway ||
+		name == HTTPRoute ||
+		name == ReferenceGrant
+}
diff --git a/projects/gateway2/wellknown/gwapi.go b/projects/gateway2/wellknown/gwapi.go
@@ -32,6 +32,10 @@ const (
 )
 
 var (
+	// SupportedVersions specifies the supported versions of Gateway API.
+	// [TODO] Remove "v1.0.0-rc1" when https://github.com/solo-io/gloo/issues/10115 is fixed.
+	SupportedVersions = []string{"v1.2.0-rc2", "v1.0.0", "v1.0.0-rc1"}
+
 	GatewayGVK = schema.GroupVersionKind{
 		Group:   GatewayGroup,
 		Version: "v1",

diff --git a/projects/gloo/pkg/api/v1/kube/wellknown/types.go b/projects/gloo/pkg/api/v1/kube/wellknown/types.go
@@ -3,13 +3,14 @@ package wellknown
 import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 )
 
 var (
 	SecretGVK         = corev1.SchemeGroupVersion.WithKind("Secret")
 	ConfigMapGVK      = corev1.SchemeGroupVersion.WithKind("ConfigMap")
 	ServiceGVK        = corev1.SchemeGroupVersion.WithKind("Service")
 	ServiceAccountGVK = corev1.SchemeGroupVersion.WithKind("ServiceAccount")
-
-	DeploymentGVK = appsv1.SchemeGroupVersion.WithKind("Deployment")
+	CrdGVK            = apiextv1.SchemeGroupVersion.WithKind("CustomResourceDefinition")
+	DeploymentGVK     = appsv1.SchemeGroupVersion.WithKind("Deployment")
 )
diff --git a/projects/gloo/pkg/servers/iosnapshot/history_test.go b/projects/gloo/pkg/servers/iosnapshot/history_test.go
@@ -13,6 +13,7 @@ import (
 	wellknownkube "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/wellknown"
 
 	corev1 "k8s.io/api/core/v1"
+	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 
 	skmatchers "github.com/solo-io/solo-kit/test/matchers"
 
@@ -186,6 +187,16 @@ var _ = Describe("History", func() {
 						"key": "value",
 					},
 				},
+				&apiextv1.CustomResourceDefinition{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "kube-crd",
+						Namespace: "crd",
+						ManagedFields: []metav1.ManagedFieldsEntry{{
+							Manager: "manager",
+						}},
+					},
+					Spec: apiextv1.CustomResourceDefinitionSpec{},
+				},
 				&apiv1.Gateway{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "kube-gw",
@@ -420,6 +431,19 @@ var _ = Describe("History", func() {
 				), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.ConfigMapGVK, "configmap", "kube-configmap"))
 			})
 
+			It("Includes CustomResourceDefinitions", func() {
+				returnedResources := getInputSnapshotObjects(ctx, history)
+				Expect(returnedResources).To(ContainElement(
+					matchers.MatchClientObject(
+						wellknownkube.CrdGVK,
+						types.NamespacedName{
+							Name:      "kube-crd",
+							Namespace: "crd",
+						},
+					),
+				), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.CrdGVK, "crd", "kube-crd"))
+			})
+
 		})
 
 		Context("Kubernetes Gateway API Resources", func() {

diff --git a/projects/gloo/pkg/servers/iosnapshot/resources.go b/projects/gloo/pkg/servers/iosnapshot/resources.go
@@ -17,6 +17,10 @@ var (
 		wellknownkube.ConfigMapGVK,
 	}
 
+	KubernetesExtGVKs = []schema.GroupVersionKind{
+		wellknownkube.CrdGVK,
+	}
+
 	GlooGVKs = []schema.GroupVersionKind{
 		gloov1.SettingsGVK,
 		gloov1.UpstreamGVK,
@@ -68,5 +72,6 @@ var (
 		EdgeOnlyInputSnapshotGVKs,
 		KubernetesGatewayGVKs,
 		KubernetesGatewayIntegrationPolicyGVKs,
+		KubernetesExtGVKs,
 	)
 )