Backport 1.17: Remove Istio trust domain (#9749)

* Remove Istio trust domain (#9713) * add trust domain * changelog * tests * fix test name * add to workflow * Adding changelog file to new location * Deleting changelog file from old location * rebalance tests, fix helm * update workflow with numbers for loadbalancing between e2e test clusters * pr feedback * update gateway proxies to be disabled for k8s gateway e2e tests * cleanup istio install, remove old trust domain tests * fix merge * minimal ci change * fix test name * fix glooctl test helm chart * fix istio edge gw manifest * fix configmap template * Adding changelog file to new location * Deleting changelog file from old location * missing gatewayProxy * t.Error * Adding changelog file to new location * Deleting changelog file from old location --------- Co-authored-by: changelog-bot <changelog-bot> Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> * move changelog --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: Jacob Bohanon <[email protected]>
solo-io · Jul 15, 2024 · 3067c26 · 3067c26
1 parent cb2b470
commit 3067c26
Show file tree

Hide file tree

Showing 24 changed files with 359 additions and 30 deletions.
diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml
@@ -59,7 +59,7 @@ jobs:
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-two'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$'
+          go-test-run-regex: '^TestK8sGatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$'
 
         # May 14th: ~ minutes execution time (see load_balancing_tests.md)
         - cluster-name: 'cluster-three'

diff --git a/changelog/v1.17.0/add-istio-trust-domain.yaml b/changelog/v1.17.0/add-istio-trust-domain.yaml
@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6472
+    resolvesIssue: false
+    description: |
+      Remove ---trust-domain arg for Istio agent proxy. Envoy no longer runs by default and this is not used by the 
+      istio-proxy for the CSR request.
diff --git a/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml b/install/helm/gloo/templates/7-gateway-proxy-deployment.yaml
@@ -393,7 +393,6 @@ spec:
         - NONE
         - --dnsRefreshRate
         - 300s
-        - --trust-domain=cluster.local
         - --controlPlaneBootstrap=false
         env:
           - name: OUTPUT_CERTS

diff --git a/install/test/values/val_custom_istio_sidecar.yaml b/install/test/values/val_custom_istio_sidecar.yaml
@@ -31,7 +31,6 @@ global:
         - 300s
         - --statusPort
         - "15021"
-        - --trust-domain=cluster.local
         - --controlPlaneBootstrap=false
         env:
           - name: OUTPUT_CERTS

diff --git a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml
@@ -173,7 +173,6 @@ spec:
           - NONE
           - --dnsRefreshRate
           - 300s
-          - --trust-domain=cluster.local
           - --controlPlaneBootstrap=false
         env:
           - name: OUTPUT_CERTS

diff --git a/projects/gloo/cli/pkg/cmd/istio/inject.go b/projects/gloo/cli/pkg/cmd/istio/inject.go
@@ -179,7 +179,7 @@ func addSdsSidecar(ctx context.Context, deployment *appsv1.Deployment, glooNames
 }
 
 // addIstioSidecar adds an Istio sidecar to the given deployment's containers
-func addIstioSidecar(ctx context.Context, deployment *appsv1.Deployment, istioNamespace string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) error {
+func addIstioSidecar(ctx context.Context, deployment *appsv1.Deployment, istioNamespace, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) error {
 	// Get current istio version & JWT policy from cluster
 	istioPilotContainer, err := getIstiodContainer(ctx, istioNamespace)
 	if err != nil {

diff --git a/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go b/projects/gloo/cli/pkg/cmd/istio/sidecars/istio.go
@@ -8,7 +8,7 @@ import (
 )
 
 // Sidecar for Istio 1.7.x releases, also works for Istio 1.8.x, 1.9.x and 1.10.x releases
-func generateIstioSidecar(version, jwtPolicy string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) *corev1.Container {
+func generateIstioSidecar(version, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) *corev1.Container {
 	sidecar := &corev1.Container{
 		Name:  "istio-proxy",
 		Image: "docker.io/istio/proxyv2:" + version,
@@ -33,7 +33,6 @@ func generateIstioSidecar(version, jwtPolicy string, istioMetaMeshID string, ist
 			"NONE",
 			"--dnsRefreshRate",
 			"300s",
-			"--trust-domain=cluster.local",
 			"--controlPlaneBootstrap=false",
 		},
 		Env: []corev1.EnvVar{

diff --git a/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go b/projects/gloo/cli/pkg/cmd/istio/sidecars/matcher.go
@@ -7,6 +7,6 @@ import (
 // GetIstioSidecar will return an Istio sidecar for the given
 // version of Istio, with the given jwtPolicy, to run
 // in the gateway-proxy pod
-func GetIstioSidecar(istioVersion, jwtPolicy string, istioMetaMeshID string, istioMetaClusterID string, istioDiscoveryAddress string) (*corev1.Container, error) {
+func GetIstioSidecar(istioVersion, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress string) (*corev1.Container, error) {
 	return generateIstioSidecar(istioVersion, jwtPolicy, istioMetaMeshID, istioMetaClusterID, istioDiscoveryAddress), nil
 }
diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go
@@ -144,6 +144,10 @@ func (i *TestInstallation) InstallMinimalIstio(ctx context.Context) error {
 	return cluster.InstallMinimalIstio(ctx, i.IstioctlBinary, i.ClusterContext.KubeContext)
 }
 
+func (i *TestInstallation) InstallRevisionedIstio(ctx context.Context) error {
+	return cluster.InstallRevisionedIstio(ctx, i.IstioctlBinary, i.ClusterContext.KubeContext, "1-22-1", "minimal")
+}
+
 func (i *TestInstallation) UninstallIstio() error {
 	return cluster.UninstallIstio(i.IstioctlBinary, i.ClusterContext.KubeContext)
 }

diff --git a/test/kubernetes/e2e/tests/automtls_istio_test.go b/test/kubernetes/e2e/tests/automtls_istio_test.go
@@ -29,7 +29,8 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) {
 	err := testInstallation.AddIstioctl(ctx)
 	if err != nil {
 		log.Printf("failed to install: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// We register the cleanup function _before_ we actually perform the installation.
@@ -50,15 +51,17 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) {
 		err = testInstallation.UninstallIstio()
 		if err != nil {
 			log.Printf("failed to uninstall: %v\n", err)
-			t.Fail()
+			// immediately stop if Istio installation fails
+			t.Error()
 		}
 	})
 
 	// Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy
 	err = testInstallation.InstallMinimalIstio(ctx)
 	if err != nil {
 		log.Printf("failed to install: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// Install Gloo Gateway

diff --git a/test/kubernetes/e2e/tests/istio_edge_api_test.go b/test/kubernetes/e2e/tests/istio_edge_api_test.go
@@ -32,7 +32,8 @@ func TestIstioEdgeApiGateway(t *testing.T) {
 	err := testInstallation.AddIstioctl(ctx)
 	if err != nil {
 		log.Printf("failed to add istioctl: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// We register the cleanup function _before_ we actually perform the installation.
@@ -53,15 +54,17 @@ func TestIstioEdgeApiGateway(t *testing.T) {
 		err = testInstallation.UninstallIstio()
 		if err != nil {
 			log.Printf("failed to uninstall: %v\n", err)
-			t.Fail()
+			// immediately stop if Istio installation fails
+			t.Error()
 		}
 	})
 
 	// Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy
 	err = testInstallation.InstallMinimalIstio(ctx)
 	if err != nil {
 		log.Printf("failed to install: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// Install Gloo Gateway with only Edge APIs enabled

diff --git a/test/kubernetes/e2e/tests/istio_regression_test.go b/test/kubernetes/e2e/tests/istio_regression_test.go
@@ -32,7 +32,8 @@ func TestIstioRegression(t *testing.T) {
 	err := testInstallation.AddIstioctl(ctx)
 	if err != nil {
 		log.Printf("failed to add istioctl: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// We register the cleanup function _before_ we actually perform the installation.
@@ -53,15 +54,17 @@ func TestIstioRegression(t *testing.T) {
 		err = testInstallation.UninstallIstio()
 		if err != nil {
 			log.Printf("failed to uninstall: %v\n", err)
-			t.Fail()
+			// immediately stop if Istio installation fails
+			t.Error()
 		}
 	})
 
 	// Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy
 	err = testInstallation.InstallMinimalIstio(ctx)
 	if err != nil {
 		log.Printf("failed to install: %v\n", err)
-		t.Fail()
+		// immediately stop if Istio installation fails
+		t.Error()
 	}
 
 	// Install Gloo Gateway with only Edge APIs enabled

diff --git a/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-automtls-k8s-gateway-test-helm.yaml
@@ -40,7 +40,7 @@ gloo:
     livenessProbeEnabled: true
 gatewayProxies:
   gatewayProxy:
-    healthyPanicThreshold: 0
+    disabled: true
 # Disable discovery, not recommended for production.
 discovery:
   enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-k8s-gateway-test-helm.yaml
@@ -40,7 +40,7 @@ gloo:
     livenessProbeEnabled: true
 gatewayProxies:
   gatewayProxy:
-    healthyPanicThreshold: 0
+    disabled: true
 # Disable discovery, not recommended for production
 discovery:
   enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml b/test/kubernetes/e2e/tests/manifests/istio-revision-helm.yaml
@@ -0,0 +1,48 @@
+global:
+  # Set up gloo with istio integration enabled (through `enableIstioSidecarOnGateway`)
+  istioIntegration:
+    enableIstioSidecarOnGateway: true
+    disableAutoinjection: true # We do not want Gloo components to be included in the mesh
+  istioSDS:
+    enabled: true
+  glooMtls:
+    istioProxy:
+      image:
+        repository: proxyv2
+        registry: docker.io/istio
+        tag: 1.22.0 # This tag has to match the version of Istio being used in the test
+  podSecurityStandards:
+    container:
+      enableRestrictedContainerDefaults: true
+gloo:
+  logLevel: info
+  disableLeaderElection: true
+  deployment:
+    #  We have limited GitHub action resources which can cause containers to not create
+    # therefore we lessen the cpu resource requests values from the default (500m) to 100m.
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+gatewayProxies:
+  gatewayProxy:
+    istioDiscoveryAddress: istiod-1-22-1.istio-system.svc:15012 # this needs to be set for Istio integration to work with revisions
+    podTemplate:
+      resources:
+        requests:
+          cpu: 100m
+          memory: 256Mi
+    healthyPanicThreshold: 0
+
+# These values are recommended production values and are not expected to impact tested behavior for the Istio suite
+settings:
+  invalidConfigPolicy:
+    replaceInvalidRoutes: true
+    invalidRouteResponseCode: 404
+    invalidRouteResponseBody: Gloo Gateway has invalid configuration.
+gateway:
+  persistProxySpec: true
+  logLevel: info
+  validation:
+    allowWarnings: true
+    alwaysAcceptResources: false
diff --git a/test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml b/test/kubernetes/e2e/tests/manifests/istio-revision-k8s-gateway.yaml
@@ -0,0 +1,51 @@
+global:
+  # Set up gloo with istio integration enabled
+  istioIntegration:
+    enabled: true
+    enableAutoMtls: false # Automtls is disabled in this test setup
+  image:
+    pullPolicy: IfNotPresent
+  # Note: glooRbac.namespaced settings are not supported with Gloo Gateway https://github.com/solo-io/solo-projects/issues/6064
+  # Gateway API fundamentally expects HTTPRoutes and Gateways in any namespace and cross-namespace references to be supported
+  # Currently we are explicitly disabled namespaced roles for Gloo Gateway tests, but this can be left unset.
+  glooRbac:
+    namespaced: false
+settings:
+  # Gloo Gateway requires access to namespaces outside of the install namespace to watch and create Gateway resources
+  # singleNamespace=false must be set for namespace watch to work correctly. See: https://github.com/solo-io/solo-projects/issues/6058
+  singleNamespace: false
+  create: true
+  invalidConfigPolicy:
+    replaceInvalidRoutes: true
+    invalidRouteResponseCode: 404
+    invalidRouteResponseBody: Gloo Gateway has invalid configuration.
+gateway:
+  persistProxySpec: false
+  logLevel: info
+  validation:
+    allowWarnings: true
+    alwaysAcceptResources: false
+    # skipping delete validation due to flakes per https://github.com/solo-io/solo-projects/issues/6272
+    webhook:
+      skipDeleteValidationResources:
+        - upstreams
+kubeGateway:
+  # This is the field that enables the K8s Gateway Integration in Gloo Gateway
+  enabled: true
+  gatewayParameters:
+    glooGateway:
+      istio:
+        istioProxyContainer:
+          istioDiscoveryAddress: istiod-1-22-1.istio-system.svc:15012 # this needs to be set for Istio integration to work with revisions
+gloo:
+  logLevel: info
+  disableLeaderElection: true
+  deployment:
+    replicas: 1
+    livenessProbeEnabled: true
+gatewayProxies:
+  gatewayProxy:
+    disabled: true
+# Disable discovery, not recommended for production
+discovery:
+  enabled: false
diff --git a/...bernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml b/...bernetes/e2e/tests/manifests/k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml
@@ -41,7 +41,7 @@ gloo:
     livenessProbeEnabled: true
 gatewayProxies:
   gatewayProxy:
-    healthyPanicThreshold: 0
+    disabled: true
 # Disable discovery, not recommended for production
 discovery:
   enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-no-webhook-validation-test-helm.yaml
@@ -38,7 +38,7 @@ gloo:
     livenessProbeEnabled: true
 gatewayProxies:
   gatewayProxy:
-    healthyPanicThreshold: 0
+    disabled: true
 # Disable discovery, not recommended for production
 discovery:
   enabled: false
diff --git a/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml b/test/kubernetes/e2e/tests/manifests/k8s-gateway-test-helm.yaml
@@ -36,6 +36,7 @@ gloo:
     livenessProbeEnabled: true
 gatewayProxies:
   gatewayProxy:
+    # TODO(npolshak): Add disabled=true here once glooctl tests are in the new framework: https://github.com/solo-io/solo-projects/issues/6303
     healthyPanicThreshold: 0
 # Disable discovery, not recommended for production
 discovery:

diff --git a/test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go b/test/kubernetes/e2e/tests/revision_istio_edge_gw_test.go
@@ -0,0 +1,76 @@
+package tests_test
+
+import (
+	"context"
+	"log"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/solo-io/gloo/test/kubernetes/e2e"
+	. "github.com/solo-io/gloo/test/kubernetes/e2e/tests"
+	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
+	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
+
+	"github.com/solo-io/skv2/codegen/util"
+)
+
+// TestRevisionIstioRegression is the function which executes a series of tests against a given installation where
+// the k8s Gateway controller is disabled and the deprecated Istio integration values are used to check for regressions
+func TestRevisionIstioRegression(t *testing.T) {
+	ctx := context.Background()
+	testInstallation := e2e.CreateTestInstallation(
+		t,
+		&gloogateway.Context{
+			InstallNamespace:   "istio-rev-regression-test",
+			ValuesManifestFile: filepath.Join(util.MustGetThisDir(), "manifests", "istio-revision-helm.yaml"),
+		},
+	)
+
+	testHelper := e2e.MustTestHelper(ctx, testInstallation)
+
+	err := testInstallation.AddIstioctl(ctx)
+	if err != nil {
+		log.Printf("failed to add istioctl: %v\n", err)
+		// immediately stop if Istio installation fails
+		t.Error()
+	}
+
+	// We register the cleanup function _before_ we actually perform the installation.
+	// This allows us to uninstall Gloo Gateway, in case the original installation only completed partially
+	t.Cleanup(func() {
+		if t.Failed() {
+			testInstallation.PreFailHandler(ctx)
+
+			// Generate istioctl bug report
+			testInstallation.CreateIstioBugReport(ctx)
+		}
+
+		testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error {
+			return testHelper.UninstallGlooAll()
+		})
+
+		// Uninstall Istio
+		err = testInstallation.UninstallIstio()
+		if err != nil {
+			log.Printf("failed to uninstall: %v\n", err)
+			// immediately stop if Istio installation fails
+			t.Error()
+		}
+	})
+
+	// Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy
+	err = testInstallation.InstallRevisionedIstio(ctx)
+	if err != nil {
+		log.Printf("failed to install: %v\n", err)
+		// immediately stop if Istio installation fails
+		t.Error()
+	}
+
+	// Install Gloo Gateway with only Edge APIs enabled
+	testInstallation.InstallGlooGateway(ctx, func(ctx context.Context) error {
+		return testHelper.InstallGloo(ctx, 5*time.Minute, helper.WithExtraArgs("--values", testInstallation.Metadata.ValuesManifestFile))
+	})
+
+	RevisionIstioEdgeGatewaySuiteRunner().Run(ctx, t, testInstallation)
+}