diff --git a/changelog/v1.18.0-beta25/issue_10092.yaml b/changelog/v1.18.0-beta25/issue_10092.yaml new file mode 100644 index 00000000000..56a8b39037c --- /dev/null +++ b/changelog/v1.18.0-beta25/issue_10092.yaml @@ -0,0 +1,7 @@ +changelog: + - type: FIX + issueLink: https://github.com/solo-io/gloo/issues/10092 + resolvesIssue: true + description: >- + Adds support for the GatewayClass SupportedVersion status condition and updates the ReconcileGatewayClasses() + method to manage status conditions according to the Gateway API specification. diff --git a/install/helm/gloo/templates/44-rbac.yaml b/install/helm/gloo/templates/44-rbac.yaml index 9b92a00eb13..6f24582d642 100644 --- a/install/helm/gloo/templates/44-rbac.yaml +++ b/install/helm/gloo/templates/44-rbac.yaml @@ -29,6 +29,11 @@ rules: resources: - endpointslices verbs: ["get", "list", "watch"] +- apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: ["get", "list", "watch"] - apiGroups: - "gateway.solo.io" resources: diff --git a/pkg/schemes/scheme.go b/pkg/schemes/scheme.go index 03b90c7f1d8..a319b5b0294 100644 --- a/pkg/schemes/scheme.go +++ b/pkg/schemes/scheme.go @@ -9,6 +9,7 @@ import ( ratelimitv1alpha1 "github.com/solo-io/solo-apis/pkg/api/ratelimit.solo.io/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" apiv1 "sigs.k8s.io/gateway-api/apis/v1" apiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -25,6 +26,9 @@ var SchemeBuilder = runtime.SchemeBuilder{ corev1.AddToScheme, appsv1.AddToScheme, + // Kubernetes apiextensions resources + apiextv1.AddToScheme, + // Solo Kubernetes Gateway API resources sologatewayv1alpha1.AddToScheme, diff --git a/projects/gateway2/controller/controller.go b/projects/gateway2/controller/controller.go index 1f368c866bd..648c80e7af8 100644 --- a/projects/gateway2/controller/controller.go +++ b/projects/gateway2/controller/controller.go @@ -3,8 +3,11 @@ package controller import ( "context" "fmt" + "strings" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -24,6 +27,7 @@ import ( sologatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1/kube/apis/gateway.solo.io/v1" "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1" + "github.com/solo-io/gloo/projects/gateway2/crds" "github.com/solo-io/gloo/projects/gateway2/deployer" "github.com/solo-io/gloo/projects/gateway2/extensions" "github.com/solo-io/gloo/projects/gateway2/query" @@ -62,13 +66,15 @@ func NewBaseGatewayController(ctx context.Context, cfg GatewayConfig) error { controllerBuilder := &controllerBuilder{ cfg: cfg, reconciler: &controllerReconciler{ - cli: cfg.Mgr.GetClient(), - scheme: cfg.Mgr.GetScheme(), - kick: cfg.Kick, + controllerName: cfg.ControllerName, + cli: cfg.Mgr.GetClient(), + scheme: cfg.Mgr.GetScheme(), + kick: cfg.Kick, }, } return run(ctx, + controllerBuilder.watchCRDs, controllerBuilder.watchGwClass, controllerBuilder.watchGw, controllerBuilder.watchHttpRoute, @@ -244,6 +250,26 @@ func shouldIgnoreStatusChild(gvk schema.GroupVersionKind) bool { return gvk.Kind == "Deployment" } +// watchCRDs sets up a controller to watch for changes to specific Gateway API CRDs and triggers +// GatewayClass reconciliation if generation or annotations change. +func (c *controllerBuilder) watchCRDs(ctx context.Context) error { + return ctrl.NewControllerManagedBy(c.cfg.Mgr). + For(&apiextv1.CustomResourceDefinition{}). + WithEventFilter(predicate.NewPredicateFuncs(func(object client.Object) bool { + // We care about changes to the specific CRDs + crd, ok := object.(*apiextv1.CustomResourceDefinition) + if !ok { + return false + } + return crds.IsSupported(crd.Name) + })). + WithEventFilter(predicate.Or( + predicate.AnnotationChangedPredicate{}, + predicate.GenerationChangedPredicate{}, + )). + Complete(reconcile.Func(c.reconciler.ReconcileCRDs)) +} + func (c *controllerBuilder) watchGwClass(_ context.Context) error { return ctrl.NewControllerManagedBy(c.cfg.Mgr). WithEventFilter(predicate.GenerationChangedPredicate{}). @@ -330,9 +356,10 @@ func (c *controllerBuilder) watchDirectResponses(_ context.Context) error { } type controllerReconciler struct { - cli client.Client - scheme *runtime.Scheme - kick func(ctx context.Context) + controllerName string + cli client.Client + scheme *runtime.Scheme + kick func(ctx context.Context) } func (r *controllerReconciler) ReconcileHttpListenerOptions(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -408,8 +435,39 @@ func (r *controllerReconciler) ReconcileReferenceGrants(ctx context.Context, req return ctrl.Result{}, nil } +func (r *controllerReconciler) ReconcileCRDs(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx).WithValues("CustomResourceDefinition", req.Name) + + log.Info("reconciling CustomResourceDefinition") + + var gatewayClassList apiv1.GatewayClassList + if err := r.cli.List(ctx, &gatewayClassList); err != nil { + log.Error(err, "Failed to list GatewayClasses") + return ctrl.Result{}, err + } + + for _, gatewayClass := range gatewayClassList.Items { + if gatewayClass.Spec.ControllerName == apiv1.GatewayController(r.controllerName) { + req := ctrl.Request{ + NamespacedName: client.ObjectKey{ + Name: gatewayClass.Name, + }, + } + + if _, err := r.ReconcileGatewayClasses(ctx, req); err != nil { + log.Error(err, "Failed to reconcile GatewayClass", "name", gatewayClass.Name) + return ctrl.Result{}, err + } + } + } + + log.Info("Reconciled CustomResourceDefinition") + + return ctrl.Result{}, nil +} + func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := log.FromContext(ctx).WithValues("gwclass", req.NamespacedName) + log := log.FromContext(ctx).WithValues("GatewayClass", req.Name) gwclass := &apiv1.GatewayClass{} if err := r.cli.Get(ctx, req.NamespacedName, gwclass); err != nil { @@ -420,31 +478,74 @@ func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req return ctrl.Result{}, client.IgnoreNotFound(err) } - log.Info("reconciling gateway class") + log.Info("Reconciling GatewayClass") - // mark it as accepted: + // Initialize the status conditions. No need to set LastTransitionTime since it's handled by SetStatusCondition. + msg := "Gateway API CRDs are a supported version" acceptedCondition := metav1.Condition{ Type: string(apiv1.GatewayClassConditionStatusAccepted), Status: metav1.ConditionTrue, Reason: string(apiv1.GatewayClassReasonAccepted), + Message: msg, ObservedGeneration: gwclass.Generation, - // no need to set LastTransitionTime, it will be set automatically by SetStatusCondition } - meta.SetStatusCondition(&gwclass.Status.Conditions, acceptedCondition) - - // TODO: This should actually check the version of the CRDs in the cluster to be 100% sure - supportedVersionCondition := metav1.Condition{ + supportedCondition := metav1.Condition{ Type: string(apiv1.GatewayClassConditionStatusSupportedVersion), Status: metav1.ConditionTrue, - ObservedGeneration: gwclass.Generation, Reason: string(apiv1.GatewayClassReasonSupportedVersion), + Message: msg, + ObservedGeneration: gwclass.Generation, + } + + // Check CRD versions + supported, err := r.checkCRDVersions(ctx) + if err != nil { + log.Error(err, "Failed to check CRD versions") + return ctrl.Result{}, err + } + + if !supported { + // Update the values of status conditions + msg = fmt.Sprintf("Unsupported Gateway API CRDs detected. Supported versions are: %s", + strings.Join(wellknown.SupportedVersions, ", ")) + acceptedCondition.Status = metav1.ConditionFalse + acceptedCondition.Reason = string(apiv1.GatewayClassReasonUnsupportedVersion) + acceptedCondition.Message = msg + supportedCondition.Status = metav1.ConditionFalse + supportedCondition.Reason = string(apiv1.GatewayClassReasonUnsupportedVersion) + supportedCondition.Message = msg } - meta.SetStatusCondition(&gwclass.Status.Conditions, supportedVersionCondition) + + // Set the status conditions + meta.SetStatusCondition(&gwclass.Status.Conditions, acceptedCondition) + meta.SetStatusCondition(&gwclass.Status.Conditions, supportedCondition) if err := r.cli.Status().Update(ctx, gwclass); err != nil { + log.Error(err, "Failed to update GatewayClass status") return ctrl.Result{}, err } - log.Info("updated gateway class status") + + log.Info("Reconciled GatewayClass") return ctrl.Result{}, nil } + +// checkCRDVersions checks that the "gateway.networking.k8s.io/bundle-version" annotation key is set +// to a supported version for each required Gateway API CRD. +func (r *controllerReconciler) checkCRDVersions(ctx context.Context) (bool, error) { + for _, crdName := range crds.Required { + crd := &apiextv1.CustomResourceDefinition{} + if err := r.cli.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + + bundleVersion, exists := crd.Annotations[crds.BundleVersionAnnotation] + if !exists || !crds.IsSupportedVersion(bundleVersion) { + return false, nil + } + } + return true, nil +} diff --git a/projects/gateway2/crds/crds.go b/projects/gateway2/crds/crds.go index d2541cee21b..4832e03440f 100644 --- a/projects/gateway2/crds/crds.go +++ b/projects/gateway2/crds/crds.go @@ -2,7 +2,43 @@ package crds import ( _ "embed" + + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/solo-io/gloo/projects/gateway2/wellknown" +) + +const ( + // BundleVersionAnnotation is the annotation key used in the Gateway API CRDs to specify + // the installed Gateway API version. + // TODO: Use CRD annotation const from sigs.k8s.io/gateway-api/pkg/consts when v1.1.0 is supported. + BundleVersionAnnotation = "gateway.networking.k8s.io/bundle-version" + // GatewayClass is the name of the GatewayClass CRD. + GatewayClass = "gatewayclasses.gateway.networking.k8s.io" + // Gateway is the name of the Gateway CRD. + Gateway = "gateways.gateway.networking.k8s.io" + // HTTPRoute is the name of the HTTPRoute CRD. + HTTPRoute = "httproutes.gateway.networking.k8s.io" + // ReferenceGrant is the name of the ReferenceGrant CRD. + ReferenceGrant = "referencegrants.gateway.networking.k8s.io" ) //go:embed gateway-crds.yaml var GatewayCrds []byte + +// Required is a list of required Gateway API CRDs. +var Required = []string{GatewayClass, Gateway, HTTPRoute, ReferenceGrant} + +// IsSupportedVersion checks if the CRD version is recognized and supported. +func IsSupportedVersion(version string) bool { + supportedVersions := sets.NewString(wellknown.SupportedVersions...) + return supportedVersions.Has(version) +} + +// IsSupported checks if the CRD is supported based on the provided name. +func IsSupported(name string) bool { + return name == GatewayClass || + name == Gateway || + name == HTTPRoute || + name == ReferenceGrant +} diff --git a/projects/gateway2/wellknown/gwapi.go b/projects/gateway2/wellknown/gwapi.go index 47eaacf6dc3..25367aca549 100644 --- a/projects/gateway2/wellknown/gwapi.go +++ b/projects/gateway2/wellknown/gwapi.go @@ -32,6 +32,10 @@ const ( ) var ( + // SupportedVersions specifies the supported versions of Gateway API. + // [TODO] Remove "v1.0.0-rc1" when https://github.com/solo-io/gloo/issues/10115 is fixed. + SupportedVersions = []string{"v1.2.0-rc2", "v1.0.0", "v1.0.0-rc1"} + GatewayGVK = schema.GroupVersionKind{ Group: GatewayGroup, Version: "v1", diff --git a/projects/gloo/pkg/api/v1/kube/wellknown/types.go b/projects/gloo/pkg/api/v1/kube/wellknown/types.go index 34ba6fde3f9..01971c4c5d5 100644 --- a/projects/gloo/pkg/api/v1/kube/wellknown/types.go +++ b/projects/gloo/pkg/api/v1/kube/wellknown/types.go @@ -3,6 +3,7 @@ package wellknown import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) var ( @@ -10,6 +11,6 @@ var ( ConfigMapGVK = corev1.SchemeGroupVersion.WithKind("ConfigMap") ServiceGVK = corev1.SchemeGroupVersion.WithKind("Service") ServiceAccountGVK = corev1.SchemeGroupVersion.WithKind("ServiceAccount") - - DeploymentGVK = appsv1.SchemeGroupVersion.WithKind("Deployment") + CrdGVK = apiextv1.SchemeGroupVersion.WithKind("CustomResourceDefinition") + DeploymentGVK = appsv1.SchemeGroupVersion.WithKind("Deployment") ) diff --git a/projects/gloo/pkg/servers/iosnapshot/history_test.go b/projects/gloo/pkg/servers/iosnapshot/history_test.go index e0209632284..67eea1bfd4c 100644 --- a/projects/gloo/pkg/servers/iosnapshot/history_test.go +++ b/projects/gloo/pkg/servers/iosnapshot/history_test.go @@ -13,6 +13,7 @@ import ( wellknownkube "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/wellknown" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" skmatchers "github.com/solo-io/solo-kit/test/matchers" @@ -186,6 +187,16 @@ var _ = Describe("History", func() { "key": "value", }, }, + &apiextv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-crd", + Namespace: "crd", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "manager", + }}, + }, + Spec: apiextv1.CustomResourceDefinitionSpec{}, + }, &apiv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: "kube-gw", @@ -420,6 +431,19 @@ var _ = Describe("History", func() { ), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.ConfigMapGVK, "configmap", "kube-configmap")) }) + It("Includes CustomResourceDefinitions", func() { + returnedResources := getInputSnapshotObjects(ctx, history) + Expect(returnedResources).To(ContainElement( + matchers.MatchClientObject( + wellknownkube.CrdGVK, + types.NamespacedName{ + Name: "kube-crd", + Namespace: "crd", + }, + ), + ), fmt.Sprintf("results should contain %v %s.%s", wellknownkube.CrdGVK, "crd", "kube-crd")) + }) + }) Context("Kubernetes Gateway API Resources", func() { diff --git a/projects/gloo/pkg/servers/iosnapshot/resources.go b/projects/gloo/pkg/servers/iosnapshot/resources.go index 65062cf444b..889082a8633 100644 --- a/projects/gloo/pkg/servers/iosnapshot/resources.go +++ b/projects/gloo/pkg/servers/iosnapshot/resources.go @@ -17,6 +17,10 @@ var ( wellknownkube.ConfigMapGVK, } + KubernetesExtGVKs = []schema.GroupVersionKind{ + wellknownkube.CrdGVK, + } + GlooGVKs = []schema.GroupVersionKind{ gloov1.SettingsGVK, gloov1.UpstreamGVK, @@ -68,5 +72,6 @@ var ( EdgeOnlyInputSnapshotGVKs, KubernetesGatewayGVKs, KubernetesGatewayIntegrationPolicyGVKs, + KubernetesExtGVKs, ) ) diff --git a/test/kubernetes/e2e/features/gatewayclass/suite.go b/test/kubernetes/e2e/features/gatewayclass/suite.go new file mode 100644 index 00000000000..9461fad62e2 --- /dev/null +++ b/test/kubernetes/e2e/features/gatewayclass/suite.go @@ -0,0 +1,183 @@ +package gatewayclass + +import ( + "context" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/solo-io/gloo/projects/gateway2/crds" + "github.com/solo-io/gloo/projects/gateway2/wellknown" + "github.com/solo-io/gloo/test/kubernetes/e2e" +) + +var _ e2e.NewSuiteFunc = NewTestingSuite + +// testingSuite is the entire Suite of tests for the "gatewayclass" feature +// The "gatewayclass" code can be found here: /projects/gateway2/controller +type testingSuite struct { + suite.Suite + + ctx context.Context + + // testInstallation contains all the metadata/utilities necessary to execute a series of tests + // against an installation of Gloo Gateway + testInstallation *e2e.TestInstallation +} + +func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + return &testingSuite{ + ctx: ctx, + testInstallation: testInst, + } +} + +// TODO: Remove test when a version of Gateway API that includes https://github.com/kubernetes-sigs/gateway-api/pull/3368 is supported +func (s *testingSuite) TestGatewayClassConditions() { + s.T().Cleanup(func() { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gwParametersManifestFile) + s.NoError(err, "can delete gatewayparameters manifest") + s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, gwParams) + + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gcManifestFile) + s.NoError(err, "can delete gatewayclass manifest") + s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, gc) + + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, supportedCrdsManifestFile) + s.NoError(err, "can apply gateway api crd manifest") + }) + + // Apply gatewayparams and gatewayclass manifests + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gwParametersManifestFile) + s.Require().NoError(err, "can apply gatewayparameters manifest") + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, gwParams) + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, gcManifestFile) + s.Require().NoError(err, "can apply gatewayclass manifest") + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, gc) + + // Assert that the gatewayclass has the expected status conditions set to true + s.Require().EventuallyWithT(func(c *assert.CollectT) { + gcName := types.NamespacedName{Name: gc.Name, Namespace: gc.Namespace} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, gcName, gc) + assert.NoError(c, err, "gatewayclass not found") + + accepted, supportedVersion := false, false + for _, conditions := range gc.Status.Conditions { + if conditions.Type == string(gwv1.GatewayClassConditionStatusAccepted) && conditions.Status == metav1.ConditionTrue { + accepted = true + } + if conditions.Type == string(gwv1.GatewayClassConditionStatusSupportedVersion) && conditions.Status == metav1.ConditionTrue { + supportedVersion = true + } + } + assert.True(c, accepted, "gatewayclass does not include expected accepted=true status conditions") + assert.True(c, supportedVersion, "gatewayclass does not include expected supportedversion=true status conditions") + }, 10*time.Second, 1*time.Second) + + // Update the bundle-version annotation of the gatewayclass crd to an unsupported version + crd := &apiextv1.CustomResourceDefinition{} + crdName := types.NamespacedName{Name: crds.GatewayClass, Namespace: "default"} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, crdName, crd) + s.Require().NoError(err, "failed to retrieve the gatewayclass crd") + s.Require().NotNil(crd.Annotations) + crd.Annotations[crds.BundleVersionAnnotation] = "v0.0.1" + err = s.testInstallation.ClusterContext.Client.Update(s.ctx, crd) + s.Require().NoError(err, "failed to update the gatewayclass crd bundle-version annotation to an unsupported version") + + // Assert that the gatewayclass has the expected status conditions set to false + s.Require().EventuallyWithT(func(c *assert.CollectT) { + gcNN := types.NamespacedName{Name: gc.Name, Namespace: gc.Namespace} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, gcNN, gc) + assert.NoError(c, err, "gatewayclass not found") + + accepted, supportedVersion := true, true + for _, conditions := range gc.Status.Conditions { + if conditions.Type == string(gwv1.GatewayClassConditionStatusAccepted) && conditions.Status == metav1.ConditionFalse { + accepted = false + } + if conditions.Type == string(gwv1.GatewayClassConditionStatusSupportedVersion) && conditions.Status == metav1.ConditionFalse { + supportedVersion = false + } + } + assert.False(c, accepted, "gatewayclass does not include expected accepted=false status conditions") + assert.False(c, supportedVersion, "gatewayclass does not include expected supportedversion=false status conditions") + }, 10*time.Second, 1*time.Second) + + // Update the bundle-version annotation of the gatewayclass crd to a supported version + crd.Annotations[crds.BundleVersionAnnotation] = wellknown.SupportedVersions[0] + err = s.testInstallation.ClusterContext.Client.Update(s.ctx, crd) + s.Require().NoError(err, "failed to update the gatewayclass crd bundle-version annotation to a supported version") + + // Assert that the gatewayclass has the expected status conditions set to true + s.Require().EventuallyWithT(func(c *assert.CollectT) { + gcNN := types.NamespacedName{Name: gc.Name, Namespace: gc.Namespace} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, gcNN, gc) + assert.NoError(c, err, "gatewayclass not found") + + accepted, supportedVersion := false, false + for _, conditions := range gc.Status.Conditions { + if conditions.Type == string(gwv1.GatewayClassConditionStatusAccepted) && conditions.Status == metav1.ConditionTrue { + accepted = true + } + if conditions.Type == string(gwv1.GatewayClassConditionStatusSupportedVersion) && conditions.Status == metav1.ConditionTrue { + supportedVersion = true + } + } + assert.True(c, accepted, "gatewayclass does not include expected accepted=true status conditions") + assert.True(c, supportedVersion, "gatewayclass does not include expected supportedversion=true status conditions") + }, 10*time.Second, 1*time.Second) + + // Remove the bundle-version annotation from the gatewayclass crd + delete(crd.Annotations, crds.BundleVersionAnnotation) + err = s.testInstallation.ClusterContext.Client.Update(s.ctx, crd) + s.Require().NoError(err, "failed to remove the gatewayclass crd bundle-version annotation") + + // Assert that the gatewayclass has the expected status conditions set to false + s.Require().EventuallyWithT(func(c *assert.CollectT) { + gcNN := types.NamespacedName{Name: gc.Name, Namespace: gc.Namespace} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, gcNN, gc) + assert.NoError(c, err, "gatewayclass not found") + + accepted, supportedVersion := true, true + for _, conditions := range gc.Status.Conditions { + if conditions.Type == string(gwv1.GatewayClassConditionStatusAccepted) && conditions.Status == metav1.ConditionFalse { + accepted = false + } + if conditions.Type == string(gwv1.GatewayClassConditionStatusSupportedVersion) && conditions.Status == metav1.ConditionFalse { + supportedVersion = false + } + } + assert.False(c, accepted, "gatewayclass does not include expected accepted=false status conditions") + assert.False(c, supportedVersion, "gatewayclass does not include expected supportedversion=false status conditions") + }, 10*time.Second, 1*time.Second) + + // Update the bundle-version annotation of the gatewayclass crd to a supported version + crd.Annotations[crds.BundleVersionAnnotation] = wellknown.SupportedVersions[1] + err = s.testInstallation.ClusterContext.Client.Update(s.ctx, crd) + s.Require().NoError(err, "failed to update the gatewayclass crd bundle-version annotation to a supported version") + + // Assert that the gatewayclass has the expected status conditions set to true + s.Require().EventuallyWithT(func(c *assert.CollectT) { + gcNN := types.NamespacedName{Name: gc.Name, Namespace: gc.Namespace} + err = s.testInstallation.ClusterContext.Client.Get(s.ctx, gcNN, gc) + assert.NoError(c, err, "gatewayclass not found") + + accepted, supportedVersion := false, false + for _, conditions := range gc.Status.Conditions { + if conditions.Type == string(gwv1.GatewayClassConditionStatusAccepted) && conditions.Status == metav1.ConditionTrue { + accepted = true + } + if conditions.Type == string(gwv1.GatewayClassConditionStatusSupportedVersion) && conditions.Status == metav1.ConditionTrue { + supportedVersion = true + } + } + assert.True(c, accepted, "gatewayclass does not include expected accepted=true status conditions") + assert.True(c, supportedVersion, "gatewayclass does not include expected supportedversion=true status conditions") + }, 10*time.Second, 1*time.Second) +} diff --git a/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayclass.yaml b/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayclass.yaml new file mode 100644 index 00000000000..ed057d9f186 --- /dev/null +++ b/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayclass.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + annotations: + labels: + gloo: kube-gateway + name: gateway-class-feature +spec: + controllerName: solo.io/gloo-gateway + parametersRef: + group: gateway.gloo.solo.io + kind: GatewayParameters + name: gateway-class-feature + namespace: gloo-system diff --git a/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayparameters.yaml b/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayparameters.yaml new file mode 100644 index 00000000000..c700d1c177c --- /dev/null +++ b/test/kubernetes/e2e/features/gatewayclass/testdata/gatewayparameters.yaml @@ -0,0 +1,28 @@ +apiVersion: gateway.gloo.solo.io/v1alpha1 +kind: GatewayParameters +metadata: + name: gateway-class-feature +spec: + kube: + deployment: + replicas: 1 + podTemplate: + extraLabels: + pod-label-key: pod-label-val + extraAnnotations: + pod-anno-key: pod-anno-val + serviceAccount: + extraLabels: + sa-label-key: sa-label-val + extraAnnotations: + sa-anno-key: sa-anno-val + envoyContainer: + bootstrap: + logLevel: debug + componentLogLevels: + upstream: debug + connection: trace + securityContext: + runAsUser: null + runAsNonRoot: false + allowPrivilegeEscalation: true diff --git a/test/kubernetes/e2e/features/gatewayclass/types.go b/test/kubernetes/e2e/features/gatewayclass/types.go new file mode 100644 index 00000000000..195f04d4ce2 --- /dev/null +++ b/test/kubernetes/e2e/features/gatewayclass/types.go @@ -0,0 +1,31 @@ +package gatewayclass + +import ( + "path/filepath" + + "github.com/solo-io/skv2/codegen/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1" +) + +var ( + gwParametersManifestFile = filepath.Join(util.MustGetThisDir(), "testdata", "gatewayparameters.yaml") + gcManifestFile = filepath.Join(util.MustGetThisDir(), "testdata", "gatewayclass.yaml") + supportedCrdsManifestFile = filepath.Join(util.MustGetThisDir(), "../../../../../projects/gateway2/crds", "gateway-crds.yaml") + + gwParams = &v1alpha1.GatewayParameters{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-class-feature", + Namespace: "default", + }, + } + + gc = &gwv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-class-feature", + Namespace: "default", + }, + } +) diff --git a/test/kubernetes/e2e/tests/k8s_gw_tests.go b/test/kubernetes/e2e/tests/k8s_gw_tests.go index c0c8846627d..68a2fd508b4 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_tests.go @@ -5,6 +5,7 @@ import ( "github.com/solo-io/gloo/test/kubernetes/e2e/features/admin_server" "github.com/solo-io/gloo/test/kubernetes/e2e/features/deployer" "github.com/solo-io/gloo/test/kubernetes/e2e/features/directresponse" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/gatewayclass" "github.com/solo-io/gloo/test/kubernetes/e2e/features/headless_svc" "github.com/solo-io/gloo/test/kubernetes/e2e/features/http_listener_options" "github.com/solo-io/gloo/test/kubernetes/e2e/features/listener_options" @@ -19,6 +20,7 @@ import ( func KubeGatewaySuiteRunner() e2e.SuiteRunner { kubeGatewaySuiteRunner := e2e.NewSuiteRunner(false) + kubeGatewaySuiteRunner.Register("GatewayClass", gatewayclass.NewTestingSuite) kubeGatewaySuiteRunner.Register("Deployer", deployer.NewTestingSuite) kubeGatewaySuiteRunner.Register("HttpListenerOptions", http_listener_options.NewTestingSuite) kubeGatewaySuiteRunner.Register("ListenerOptions", listener_options.NewTestingSuite) diff --git a/test/kubernetes/testutils/clients/clients.go b/test/kubernetes/testutils/clients/clients.go index 4bdefb78d19..f5550c57b84 100644 --- a/test/kubernetes/testutils/clients/clients.go +++ b/test/kubernetes/testutils/clients/clients.go @@ -8,6 +8,7 @@ import ( glooinstancev1 "github.com/solo-io/solo-apis/pkg/api/fed.solo.io/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" v1 "sigs.k8s.io/gateway-api/apis/v1" @@ -36,6 +37,9 @@ func MustClientScheme() *runtime.Scheme { err = appsv1.AddToScheme(clientScheme) mustNotError(err) + err = apiextv1.AddToScheme(clientScheme) + mustNotError(err) + // Gloo resources err = gloov1.AddToScheme(clientScheme) mustNotError(err)