From 3067c264aa2025a31c7de82b8878b388d5bd0c4b Mon Sep 17 00:00:00 2001 From: Nina Polshakova Date: Mon, 15 Jul 2024 12:59:33 -0400 Subject: [PATCH] Backport 1.17: Remove Istio trust domain (#9749) * Remove Istio trust domain (#9713) * add trust domain * changelog * tests * fix test name * add to workflow * Adding changelog file to new location * Deleting changelog file from old location * rebalance tests, fix helm * update workflow with numbers for loadbalancing between e2e test clusters * pr feedback * update gateway proxies to be disabled for k8s gateway e2e tests * cleanup istio install, remove old trust domain tests * fix merge * minimal ci change * fix test name * fix glooctl test helm chart * fix istio edge gw manifest * fix configmap template * Adding changelog file to new location * Deleting changelog file from old location * missing gatewayProxy * t.Error * Adding changelog file to new location * Deleting changelog file from old location --------- Co-authored-by: changelog-bot Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> * move changelog --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: Jacob Bohanon --- .github/workflows/pr-kubernetes-tests.yaml | 2 +- changelog/v1.17.0/add-istio-trust-domain.yaml | 7 ++ .../templates/7-gateway-proxy-deployment.yaml | 1 - .../test/values/val_custom_istio_sidecar.yaml | 1 - .../templates/gateway/proxy-deployment.yaml | 1 - projects/gloo/cli/pkg/cmd/istio/inject.go | 2 +- .../gloo/cli/pkg/cmd/istio/sidecars/istio.go | 3 +- .../cli/pkg/cmd/istio/sidecars/matcher.go | 2 +- test/kubernetes/e2e/test.go | 4 + .../e2e/tests/automtls_istio_test.go | 9 ++- .../e2e/tests/istio_edge_api_test.go | 9 ++- .../e2e/tests/istio_regression_test.go | 9 ++- .../istio-automtls-k8s-gateway-test-helm.yaml | 2 +- .../istio-k8s-gateway-test-helm.yaml | 2 +- .../tests/manifests/istio-revision-helm.yaml | 48 ++++++++++++ .../manifests/istio-revision-k8s-gateway.yaml | 51 +++++++++++++ ...l-default-gatewayparameters-test-helm.yaml | 2 +- ...teway-no-webhook-validation-test-helm.yaml | 2 +- .../manifests/k8s-gateway-test-helm.yaml | 1 + .../e2e/tests/revision_istio_edge_gw_test.go | 76 +++++++++++++++++++ .../e2e/tests/revision_istio_edge_gw_tests.go | 14 ++++ .../e2e/tests/revision_istio_k8s_gw_test.go | 67 ++++++++++++++++ .../e2e/tests/revision_istio_k8s_gw_tests.go | 14 ++++ test/kubernetes/testutils/cluster/istio.go | 60 ++++++++++++--- 24 files changed, 359 insertions(+), 30 deletions(-) create mode 100644 changelog/v1.17.0/add-istio-trust-domain.yaml create mode 100644 test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml create mode 100644 test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml create mode 100644 test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go create mode 100644 test/kubernetes/e2e/tests/revision_istio_edge_gw_tests.go create mode 100644 test/kubernetes/e2e/tests/revision_istio_k8s_gw_test.go create mode 100644 test/kubernetes/e2e/tests/revision_istio_k8s_gw_tests.go diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml index 6a7a60625b3..a6a73b0f05c 100644 --- a/.github/workflows/pr-kubernetes-tests.yaml +++ b/.github/workflows/pr-kubernetes-tests.yaml @@ -59,7 +59,7 @@ jobs: # May 14th: ~ minutes execution time (see load_balancing_tests.md) - cluster-name: 'cluster-two' go-test-args: '-v -timeout=25m' - go-test-run-regex: '^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$' + go-test-run-regex: '^TestK8sGatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$' # May 14th: ~ minutes execution time (see load_balancing_tests.md) - cluster-name: 'cluster-three' diff --git a/changelog/v1.17.0/add-istio-trust-domain.yaml b/changelog/v1.17.0/add-istio-trust-domain.yaml new file mode 100644 index 00000000000..d4cb7b91016 --- /dev/null +++ b/changelog/v1.17.0/add-istio-trust-domain.yaml @@ -0,0 +1,7 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/6472 + resolvesIssue: false + description: | + Remove ---trust-domain arg for Istio agent proxy. Envoy no longer runs by default and this is not used by the + istio-proxy for the CSR request. diff --git a/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml b/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml index 05db521cd97..5a7179d0cb6 100644 --- a/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml +++ b/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml @@ -393,7 +393,6 @@ spec: - NONE - --dnsRefreshRate - 300s - - --trust-domain=cluster.local - --controlPlaneBootstrap=false env: - name: OUTPUT_CERTS diff --git a/install/test/values/val_custom_istio_sidecar.yaml b/install/test/values/val_custom_istio_sidecar.yaml index 905628211e9..5e871ab2f6d 100644 --- a/install/test/values/val_custom_istio_sidecar.yaml +++ b/install/test/values/val_custom_istio_sidecar.yaml @@ -31,7 +31,6 @@ global: - 300s - --statusPort - "15021" - - --trust-domain=cluster.local - --controlPlaneBootstrap=false env: - name: OUTPUT_CERTS diff --git a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml index f4f9ac5343c..b393db69080 100644 --- a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml +++ b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml @@ -173,7 +173,6 @@ spec: - NONE - --dnsRefreshRate - 300s - - --trust-domain=cluster.local - --controlPlaneBootstrap=false env: - name: OUTPUT_CERTS diff --git a/projects/gloo/cli/pkg/cmd/istio/inject.go b/projects/gloo/cli/pkg/cmd/istio/inject.go index bd93dafcde4..a21d412393f 100644 --- a/projects/gloo/cli/pkg/cmd/istio/inject.go +++ b/projects/gloo/cli/pkg/cmd/istio/inject.go @@ -179,7 +179,7 @@ func addSdsSidecar(ctx context.Context, deployment *appsv1.Deployment, glooNames } // addIstioSidecar adds an Istio sidecar to the given deployment's containers -func addIstioSidecar(ctx context.Context, deployment *appsv1.Deployment, istioNamespace string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) error { +func addIstioSidecar(ctx context.Context, deployment *appsv1.Deployment, istioNamespace, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) error { // Get current istio version & JWT policy from cluster istioPilotContainer, err := getIstiodContainer(ctx, istioNamespace) if err != nil { diff --git a/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go b/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go index b7c21dbef47..a453c092dfb 100644 --- a/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go +++ b/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go @@ -8,7 +8,7 @@ import ( ) // Sidecar for Istio 1.7.x releases, also works for Istio 1.8.x, 1.9.x and 1.10.x releases -func generateIstioSidecar(version, jwtPolicy string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) *corev1.Container { +func generateIstioSidecar(version, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) *corev1.Container { sidecar := &corev1.Container{ Name: "istio-proxy", Image: "docker.io/istio/proxyv2:" + version, @@ -33,7 +33,6 @@ func generateIstioSidecar(version, jwtPolicy string, istioMetaMeshID string, ist "NONE", "--dnsRefreshRate", "300s", - "--trust-domain=cluster.local", "--controlPlaneBootstrap=false", }, Env: []corev1.EnvVar{ diff --git a/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go b/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go index 595bdd3f0c6..cbca0b8a611 100644 --- a/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go +++ b/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go @@ -7,6 +7,6 @@ import ( // GetIstioSidecar will return an Istio sidecar for the given // version of Istio, with the given jwtPolicy, to run // in the gateway-proxy pod -func GetIstioSidecar(istioVersion, jwtPolicy string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) (*corev1.Container, error) { +func GetIstioSidecar(istioVersion, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) (*corev1.Container, error) { return generateIstioSidecar(istioVersion, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress), nil } diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index b8949ad718f..3238a381915 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -144,6 +144,10 @@ func (i *TestInstallation) InstallMinimalIstio(ctx context.Context) error { return cluster.InstallMinimalIstio(ctx, i.IstioctlBinary, i.ClusterContext.KubeContext) } +func (i *TestInstallation) InstallRevisionedIstio(ctx context.Context) error { + return cluster.InstallRevisionedIstio(ctx, i.IstioctlBinary, i.ClusterContext.KubeContext, "1-22-1", "minimal") +} + func (i *TestInstallation) UninstallIstio() error { return cluster.UninstallIstio(i.IstioctlBinary, i.ClusterContext.KubeContext) } diff --git a/test/kubernetes/e2e/tests/automtls_istio_test.go b/test/kubernetes/e2e/tests/automtls_istio_test.go index 1e2c7a67fb5..ef3e644dbf5 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_test.go +++ b/test/kubernetes/e2e/tests/automtls_istio_test.go @@ -29,7 +29,8 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { err := testInstallation.AddIstioctl(ctx) if err != nil { log.Printf("failed to install: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // We register the cleanup function _before_ we actually perform the installation. @@ -50,7 +51,8 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { err = testInstallation.UninstallIstio() if err != nil { log.Printf("failed to uninstall: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } }) @@ -58,7 +60,8 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { err = testInstallation.InstallMinimalIstio(ctx) if err != nil { log.Printf("failed to install: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // Install Gloo Gateway diff --git a/test/kubernetes/e2e/tests/istio_edge_api_test.go b/test/kubernetes/e2e/tests/istio_edge_api_test.go index a462c2ea639..274d006bc7f 100644 --- a/test/kubernetes/e2e/tests/istio_edge_api_test.go +++ b/test/kubernetes/e2e/tests/istio_edge_api_test.go @@ -32,7 +32,8 @@ func TestIstioEdgeApiGateway(t *testing.T) { err := testInstallation.AddIstioctl(ctx) if err != nil { log.Printf("failed to add istioctl: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // We register the cleanup function _before_ we actually perform the installation. @@ -53,7 +54,8 @@ func TestIstioEdgeApiGateway(t *testing.T) { err = testInstallation.UninstallIstio() if err != nil { log.Printf("failed to uninstall: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } }) @@ -61,7 +63,8 @@ func TestIstioEdgeApiGateway(t *testing.T) { err = testInstallation.InstallMinimalIstio(ctx) if err != nil { log.Printf("failed to install: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // Install Gloo Gateway with only Edge APIs enabled diff --git a/test/kubernetes/e2e/tests/istio_regression_test.go b/test/kubernetes/e2e/tests/istio_regression_test.go index 7bfd609555d..8acfd96a1f6 100644 --- a/test/kubernetes/e2e/tests/istio_regression_test.go +++ b/test/kubernetes/e2e/tests/istio_regression_test.go @@ -32,7 +32,8 @@ func TestIstioRegression(t *testing.T) { err := testInstallation.AddIstioctl(ctx) if err != nil { log.Printf("failed to add istioctl: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // We register the cleanup function _before_ we actually perform the installation. @@ -53,7 +54,8 @@ func TestIstioRegression(t *testing.T) { err = testInstallation.UninstallIstio() if err != nil { log.Printf("failed to uninstall: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } }) @@ -61,7 +63,8 @@ func TestIstioRegression(t *testing.T) { err = testInstallation.InstallMinimalIstio(ctx) if err != nil { log.Printf("failed to install: %v\n", err) - t.Fail() + // immediately stop if Istio installation fails + t.Error() } // Install Gloo Gateway with only Edge APIs enabled diff --git a/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml index 3d9ccd5ed3b..63c99d082e7 100644 --- a/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml @@ -40,7 +40,7 @@ gloo: livenessProbeEnabled: true gatewayProxies: gatewayProxy: - healthyPanicThreshold: 0 + disabled: true # Disable discovery, not recommended for production. discovery: enabled: false diff --git a/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml index 7bd52c33100..2812b314a15 100644 --- a/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml @@ -40,7 +40,7 @@ gloo: livenessProbeEnabled: true gatewayProxies: gatewayProxy: - healthyPanicThreshold: 0 + disabled: true # Disable discovery, not recommended for production discovery: enabled: false diff --git a/test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml new file mode 100644 index 00000000000..7fb610c0d33 --- /dev/null +++ b/test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml @@ -0,0 +1,48 @@ +global: + # Set up gloo with istio integration enabled (through `enableIstioSidecarOnGateway`) + istioIntegration: + enableIstioSidecarOnGateway: true + disableAutoinjection: true # We do not want Gloo components to be included in the mesh + istioSDS: + enabled: true + glooMtls: + istioProxy: + image: + repository: proxyv2 + registry: docker.io/istio + tag: 1.22.0 # This tag has to match the version of Istio being used in the test + podSecurityStandards: + container: + enableRestrictedContainerDefaults: true +gloo: + logLevel: info + disableLeaderElection: true + deployment: + # We have limited GitHub action resources which can cause containers to not create + # therefore we lessen the cpu resource requests values from the default (500m) to 100m. + resources: + requests: + cpu: 100m + memory: 256Mi +gatewayProxies: + gatewayProxy: + istioDiscoveryAddress: istiod-1-22-1.istio-system.svc:15012 # this needs to be set for Istio integration to work with revisions + podTemplate: + resources: + requests: + cpu: 100m + memory: 256Mi + healthyPanicThreshold: 0 + +# These values are recommended production values and are not expected to impact tested behavior for the Istio suite +settings: + invalidConfigPolicy: + replaceInvalidRoutes: true + invalidRouteResponseCode: 404 + invalidRouteResponseBody: Gloo Gateway has invalid configuration. +gateway: + persistProxySpec: true + logLevel: info + validation: + allowWarnings: true + alwaysAcceptResources: false \ No newline at end of file diff --git a/test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml b/test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml new file mode 100644 index 00000000000..4b5887669f7 --- /dev/null +++ b/test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml @@ -0,0 +1,51 @@ +global: + # Set up gloo with istio integration enabled + istioIntegration: + enabled: true + enableAutoMtls: false # Automtls is disabled in this test setup + image: + pullPolicy: IfNotPresent + # Note: glooRbac.namespaced settings are not supported with Gloo Gateway https://github.com/solo-io/solo-projects/issues/6064 + # Gateway API fundamentally expects HTTPRoutes and Gateways in any namespace and cross-namespace references to be supported + # Currently we are explicitly disabled namespaced roles for Gloo Gateway tests, but this can be left unset. + glooRbac: + namespaced: false +settings: + # Gloo Gateway requires access to namespaces outside of the install namespace to watch and create Gateway resources + # singleNamespace=false must be set for namespace watch to work correctly. See: https://github.com/solo-io/solo-projects/issues/6058 + singleNamespace: false + create: true + invalidConfigPolicy: + replaceInvalidRoutes: true + invalidRouteResponseCode: 404 + invalidRouteResponseBody: Gloo Gateway has invalid configuration. +gateway: + persistProxySpec: false + logLevel: info + validation: + allowWarnings: true + alwaysAcceptResources: false + # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272 + webhook: + skipDeleteValidationResources: + - upstreams +kubeGateway: + # This is the field that enables the K8s Gateway Integration in Gloo Gateway + enabled: true + gatewayParameters: + glooGateway: + istio: + istioProxyContainer: + istioDiscoveryAddress: istiod-1-22-1.istio-system.svc:15012 # this needs to be set for Istio integration to work with revisions +gloo: + logLevel: info + disableLeaderElection: true + deployment: + replicas: 1 + livenessProbeEnabled: true +gatewayProxies: + gatewayProxy: + disabled: true +# Disable discovery, not recommended for production +discovery: + enabled: false diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml index b355808189d..ff6392f920b 100644 --- a/test/kubernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml @@ -41,7 +41,7 @@ gloo: livenessProbeEnabled: true gatewayProxies: gatewayProxy: - healthyPanicThreshold: 0 + disabled: true # Disable discovery, not recommended for production discovery: enabled: false diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml index bf168e738ae..58b85205f20 100644 --- a/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml @@ -38,7 +38,7 @@ gloo: livenessProbeEnabled: true gatewayProxies: gatewayProxy: - healthyPanicThreshold: 0 + disabled: true # Disable discovery, not recommended for production discovery: enabled: false diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml index 8e9c974829d..06028cd09c9 100644 --- a/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml +++ b/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml @@ -36,6 +36,7 @@ gloo: livenessProbeEnabled: true gatewayProxies: gatewayProxy: + # TODO(npolshak): Add disabled=true here once glooctl tests are in the new framework: https://github.com/solo-io/solo-projects/issues/6303 healthyPanicThreshold: 0 # Disable discovery, not recommended for production discovery: diff --git a/test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go b/test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go new file mode 100644 index 00000000000..6244dbe43bb --- /dev/null +++ b/test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go @@ -0,0 +1,76 @@ +package tests_test + +import ( + "context" + "log" + "path/filepath" + "testing" + "time" + + "github.com/solo-io/gloo/test/kubernetes/e2e" + . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" + "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" + + "github.com/solo-io/skv2/codegen/util" +) + +// TestRevisionIstioRegression is the function which executes a series of tests against a given installation where +// the k8s Gateway controller is disabled and the deprecated Istio integration values are used to check for regressions +func TestRevisionIstioRegression(t *testing.T) { + ctx := context.Background() + testInstallation := e2e.CreateTestInstallation( + t, + &gloogateway.Context{ + InstallNamespace: "istio-rev-regression-test", + ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-revision-helm.yaml"), + }, + ) + + testHelper := e2e.MustTestHelper(ctx, testInstallation) + + err := testInstallation.AddIstioctl(ctx) + if err != nil { + log.Printf("failed to add istioctl: %v\n", err) + // immediately stop if Istio installation fails + t.Error() + } + + // We register the cleanup function _before_ we actually perform the installation. + // This allows us to uninstall Gloo Gateway, in case the original installation only completed partially + t.Cleanup(func() { + if t.Failed() { + testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) + } + + testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.UninstallGlooAll() + }) + + // Uninstall Istio + err = testInstallation.UninstallIstio() + if err != nil { + log.Printf("failed to uninstall: %v\n", err) + // immediately stop if Istio installation fails + t.Error() + } + }) + + // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy + err = testInstallation.InstallRevisionedIstio(ctx) + if err != nil { + log.Printf("failed to install: %v\n", err) + // immediately stop if Istio installation fails + t.Error() + } + + // Install Gloo Gateway with only Edge APIs enabled + testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + }) + + RevisionIstioEdgeGatewaySuiteRunner().Run(ctx, t, testInstallation) +} diff --git a/test/kubernetes/e2e/tests/revision_istio_edge_gw_tests.go b/test/kubernetes/e2e/tests/revision_istio_edge_gw_tests.go new file mode 100644 index 00000000000..6fbca49c822 --- /dev/null +++ b/test/kubernetes/e2e/tests/revision_istio_edge_gw_tests.go @@ -0,0 +1,14 @@ +package tests + +import ( + "github.com/solo-io/gloo/test/kubernetes/e2e" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/istio" +) + +func RevisionIstioEdgeGatewaySuiteRunner() e2e.SuiteRunner { + revisionIstioSuiteRunner := e2e.NewSuiteRunner(false) + + revisionIstioSuiteRunner.Register("IstioIntegration", istio.NewGlooTestingSuite) + + return revisionIstioSuiteRunner +} diff --git a/test/kubernetes/e2e/tests/revision_istio_k8s_gw_test.go b/test/kubernetes/e2e/tests/revision_istio_k8s_gw_test.go new file mode 100644 index 00000000000..9b55c412765 --- /dev/null +++ b/test/kubernetes/e2e/tests/revision_istio_k8s_gw_test.go @@ -0,0 +1,67 @@ +package tests_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/solo-io/gloo/test/kubernetes/e2e" + . "github.com/solo-io/gloo/test/kubernetes/e2e/tests" + "github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway" + "github.com/solo-io/gloo/test/kubernetes/testutils/helper" + "github.com/solo-io/skv2/codegen/util" +) + +// TestK8sGatewayIstioRevision is the function which executes a series of tests against a given installation with +// k8s gateway enabled and Istio installed with revisions +func TestK8sGatewayIstioRevision(t *testing.T) { + ctx := context.Background() + testInstallation := e2e.CreateTestInstallation( + t, + &gloogateway.Context{ + InstallNamespace: "istio-rev-k8s-gw-test", + ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-revision-k8s-gateway.yaml"), + }, + ) + + testHelper := e2e.MustTestHelper(ctx, testInstallation) + err := testInstallation.AddIstioctl(ctx) + if err != nil { + t.Fatalf("failed to get istioctl: %v", err) + } + + // We register the cleanup function _before_ we actually perform the installation. + // This allows us to uninstall Gloo Gateway, in case the original installation only completed partially + t.Cleanup(func() { + if t.Failed() { + testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) + } + + testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.UninstallGlooAll() + }) + + // Uninstall Istio + err = testInstallation.UninstallIstio() + if err != nil { + t.Fatalf("failed to uninstall: %v\n", err) + } + }) + + // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy + err = testInstallation.InstallRevisionedIstio(ctx) + if err != nil { + t.Fatalf("failed to install: %v", err) + } + + // Install Gloo Gateway + testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error { + return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile)) + }) + + RevisionIstioK8sGatewaySuiteRunner().Run(ctx, t, testInstallation) +} diff --git a/test/kubernetes/e2e/tests/revision_istio_k8s_gw_tests.go b/test/kubernetes/e2e/tests/revision_istio_k8s_gw_tests.go new file mode 100644 index 00000000000..bcd22f5a188 --- /dev/null +++ b/test/kubernetes/e2e/tests/revision_istio_k8s_gw_tests.go @@ -0,0 +1,14 @@ +package tests + +import ( + "github.com/solo-io/gloo/test/kubernetes/e2e" + "github.com/solo-io/gloo/test/kubernetes/e2e/features/istio" +) + +func RevisionIstioK8sGatewaySuiteRunner() e2e.SuiteRunner { + istioSuiteRunner := e2e.NewSuiteRunner(false) + + istioSuiteRunner.Register("IstioIntegration", istio.NewTestingSuite) + + return istioSuiteRunner +} diff --git a/test/kubernetes/testutils/cluster/istio.go b/test/kubernetes/testutils/cluster/istio.go index 6be6d1d6253..41b3aef3c22 100644 --- a/test/kubernetes/testutils/cluster/istio.go +++ b/test/kubernetes/testutils/cluster/istio.go @@ -15,6 +15,11 @@ import ( "github.com/solo-io/go-utils/contextutils" ) +const ( + // TODO(npolshak): Add support for other profiles (ambient, etc.) + minimalProfile = "minimal" +) + func GetIstioctl(ctx context.Context) (string, error) { // Download istioctl binary istioctlBinary, err := downloadIstio(ctx, getIstioVersion()) @@ -30,10 +35,53 @@ func InstallMinimalIstio( ctx context.Context, istioctlBinary, kubeContext string, ) error { + operatorFileContent := generateIstioOperatorFileContent("", minimalProfile) + operatorFile := "/tmp/istio-operator.yaml" + + err := os.WriteFile(operatorFile, []byte(operatorFileContent), 0644) + if err != nil { + return fmt.Errorf("failed to write operator file: %w", err) + } + return installIstioOperator(ctx, istioctlBinary, kubeContext, "") } -// TODO(npolshak): Add additional Istio setup options as needed (versions, revisions, ambient, etc.) +func InstallRevisionedIstio( + ctx context.Context, + istioctlBinary, kubeContext, revision, profile string, +) error { + operatorFileContent := generateIstioOperatorFileContent(revision, profile) + operatorFile := "/tmp/istio-operator.yaml" + + err := os.WriteFile(operatorFile, []byte(operatorFileContent), 0644) + if err != nil { + return fmt.Errorf("failed to write operator file: %w", err) + } + + return installIstioOperator(ctx, istioctlBinary, kubeContext, operatorFile) +} + +// TODO(npolshak): Add Istio dependency to define operator in code instead of writing file +func generateIstioOperatorFileContent(revision, profile string) string { + // use minimal as the default profile if none is provided + if profile == "" { + profile = minimalProfile + } + + baseContent := ` +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +metadata: + namespace: istio-system +spec: + profile: %s +` + if revision != "" { + return fmt.Sprintf(baseContent+" revision: %s\n", profile, revision) + } + return fmt.Sprintf(baseContent, profile) +} + func installIstioOperator( ctx context.Context, istioctlBinary, kubeContext, operatorFile string) error { @@ -41,14 +89,8 @@ func installIstioOperator( return nil } - var cmd *exec.Cmd - if operatorFile == "" { - // use the minimal profile by default if no operator file is provided - // yes | istioctl install --context --set profile=minimal - cmd = exec.Command("sh", "-c", "yes | "+istioctlBinary+" install --context "+kubeContext+" --set profile=minimal") - } else { - cmd = exec.Command("sh", "-c", "yes | "+istioctlBinary, "install", "-y", "--context", kubeContext, "-f", operatorFile) - } + // istioctl install -y --context -f + cmd := exec.Command(istioctlBinary, "install", "-y", "--context", kubeContext, "-f", operatorFile) if err := cmd.Run(); err != nil { return fmt.Errorf("istioctl install failed: %w", err)