Add Istio strict/permissive auth e2e tests for Gloo Edge Gateway apis…

… using new testing framework (#9436)

* wip

* wip

* fix istio install

* use global.istioIntegration.enableAutoMtls

* fix imports

* check settings

* fix istio uninstall

* split istio setup

* remove settings client

* pr feedback

* fixed assertions, cleanup

* changelog, debug instructions

* switch to sep headless svc test suites

* classic edge automtls tests

* k8s gw headless svc test specific suite

* regen

* add edge api tests

* passing tests

* passing glooctl tests

* fix uninject

* go mod

* changelog

* mimimize diff

* Adding changelog file to new location

* Deleting changelog file from old location

* glooctl

* use testInst.GeneratedFiles.TempDir

* remove glooctl path

* pr feedback

* regen

* pr feedback, regen

* add TestGlooctlIstioInjectEdgeApiGateway to test to run in ci

* add TestAutomtlsIstioEdgeApisGateway and TestIstioEdgeApiGateway to ci tests

* remove TestCluster refs

* fix filepath

* remove --istio-namespace, not required for uninject, use default for inject

* fix manifest file name

* add missing istio installation to automtls test

* use edge api headless svc test

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

.github/workflows/pr-kubernetes-tests.yaml

@@ -71,7 +71,7 @@ jobs:
           kubectl-version: 'v1.28.4'
           helm-version: 'v3.13.2'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestK8sGatewayIstio$$|^TestClassicEdgeGateway$$)'
+          go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$)'
 
     steps:
     - id: auto-succeed-tests

changelog/v1.17.0-beta27/add-edge-gateway-e2e-test.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/6048
+    resolvesIssue: false
+    description: >-
+      Add Gloo Edge Gateway api e2e test for Istio integration with strict/permissive peer auth. Add glooctl inject/uninject
+      test for Istio integration.

projects/gloo/cli/pkg/cmd/istio/constants.go

@@ -0,0 +1,6 @@
+package istio
+
+const (
+	SDSContainerName = "sds"
+	IstioProxyName   = "istio-proxy"
+)

projects/gloo/cli/pkg/cmd/istio/inject.go

@@ -135,10 +135,10 @@ func istioInject(_ []string, opts *options.Options) error {
 			// Check if sidecars already exist
 			if len(containers) > 1 {
 				for _, container := range containers {
-					if container.Name == "sds" {
+					if container.Name == SDSContainerName {
 						return ErrSdsAlreadyPresent
 					}
-					if container.Name == "istio-proxy" {
+					if container.Name == IstioProxyName {
 						return ErrIstioAlreadyPresent
 					}
 				}

projects/gloo/cli/pkg/cmd/istio/uninject.go

@@ -119,12 +119,12 @@ func istioUninject(_ []string, opts *options.Options) error {
 			if len(containers) > 1 {
 				for i := len(containers) - 1; i >= 0; i-- {
 					container := containers[i]
-					if container.Name == "sds" {
+					if container.Name == SDSContainerName {
 						sdsPresent = true
 						copy(containers[i:], containers[i+1:])
 						containers = containers[:len(containers)-1]
 					}
-					if container.Name == "istio-proxy" {
+					if container.Name == IstioProxyName {
 						istioPresent = true
 
 						copy(containers[i:], containers[i+1:])

projects/gloo/cli/pkg/testutils/cli.go

@@ -64,3 +64,25 @@ func (c *GlooCli) DebugLogs(ctx context.Context, extraArgs ...string) error {
 	}, extraArgs...)
 	return ExecuteCommandWithArgs(c.NewCommand(ctx), debugLogsArgs...)
 }
+
+// IstioInject inject istio-proxy and sds
+func (c *GlooCli) IstioInject(ctx context.Context, installNamespace, kubeContext string, extraArgs ...string) (string, error) {
+	injectArgs := append([]string{
+		"istio", "inject",
+		"--namespace", installNamespace,
+		"--kube-context", kubeContext,
+	}, extraArgs...)
+
+	return ExecuteCommandWithArgsOut(c.NewCommand(ctx), injectArgs...)
+}
+
+// IstioUninject uninjects istio-proxy and sds
+func (c *GlooCli) IstioUninject(ctx context.Context, installNamespace, kubeContext string, extraArgs ...string) (string, error) {
+	uninjectArgs := append([]string{
+		"istio", "uninject",
+		"--namespace", installNamespace,
+		"--kube-context", kubeContext,
+	}, extraArgs...)
+
+	return ExecuteCommandWithArgsOut(c.NewCommand(ctx), uninjectArgs...)
+}

test/gomega/matchers/pod.go

@@ -0,0 +1,46 @@
+package matchers
+
+import (
+	"fmt"
+
+	"github.com/onsi/gomega/types"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// ExpectedPod is a struct that represents the expected pod.
+type ExpectedPod struct {
+	// ContainerName is the name of the container. Required.
+	ContainerName string
+
+	// TODO(npolshak): Add more fields to match on as needed
+}
+
+// PodMatches returns a GomegaMatcher that checks whether a pod
+func PodMatches(pod ExpectedPod) types.GomegaMatcher {
+	return &podMatcher{expectedPod: pod}
+}
+
+type podMatcher struct {
+	expectedPod ExpectedPod
+}
+
+func (pm *podMatcher) Match(actual interface{}) (bool, error) {
+	pod, ok := actual.(corev1.Pod)
+	if !ok {
+		return false, fmt.Errorf("expected a pod, got %T", actual)
+	}
+	for _, container := range pod.Spec.Containers {
+		if container.Name == pm.expectedPod.ContainerName {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (pm *podMatcher) FailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected pod to have container '%s', but it was not found", pm.expectedPod.ContainerName)
+}
+
+func (pm *podMatcher) NegatedFailureMessage(actual interface{}) string {
+	return fmt.Sprintf("Expected pod not to have container '%s', but it was found", pm.expectedPod.ContainerName)
+}

test/kubernetes/e2e/features/glooctl/istio_inject_suite.go

@@ -0,0 +1,52 @@
+package glooctl
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/gomega"
+	"github.com/solo-io/gloo/projects/gateway/pkg/defaults"
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/istio"
+	"github.com/solo-io/gloo/test/gomega/matchers"
+	"github.com/solo-io/gloo/test/kubernetes/e2e"
+	"github.com/stretchr/testify/suite"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// istioInjectTestingSuite is the entire Suite of tests for the "glooctl istio inject" integration cases
+// NOTE: This suite is not intended to be run as a standalone test suite. It applies the "glooctl istio inject" command
+// to an existing installation of Gloo Gateway and verifies that the necessary resources are created, but does not clean
+// up the state after the `inject` command is run. To clean up the state run the `istioUninjectTestingSuite` after this.
+type istioInjectTestingSuite struct {
+	suite.Suite
+
+	ctx context.Context
+
+	// testInstallation contains all the metadata/utilities necessary to execute a series of tests
+	// against an installation of Gloo Gateway
+	testInstallation *e2e.TestInstallation
+}
+
+func NewIstioInjectTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
+	return &istioInjectTestingSuite{
+		ctx:              ctx,
+		testInstallation: testInst,
+	}
+}
+
+func (s *istioInjectTestingSuite) TestCanInject() {
+	// Inject istio with glooctl
+	out, err := s.testInstallation.Actions.Glooctl().IstioInject(s.ctx, s.testInstallation.Metadata.InstallNamespace, s.testInstallation.ClusterContext.KubeContext)
+	s.Assert().NoError(err, "Failed to inject istio")
+	s.Assert().Contains(out, "Istio injection was successful!")
+
+	matcher := gomega.And(
+		matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.SDSContainerName}),
+		matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.IstioProxyName}),
+	)
+	s.testInstallation.Assertions.EventuallyPodsMatches(s.ctx,
+		s.testInstallation.Metadata.InstallNamespace,
+		metav1.ListOptions{LabelSelector: fmt.Sprintf("gloo=%s", defaults.GatewayProxyName)},
+		matcher,
+	)
+}

test/kubernetes/e2e/features/glooctl/istio_uninject_suite.go

@@ -0,0 +1,53 @@
+package glooctl
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/gomega"
+	"github.com/solo-io/gloo/projects/gateway/pkg/defaults"
+	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/istio"
+	"github.com/solo-io/gloo/test/gomega/matchers"
+	"github.com/solo-io/gloo/test/kubernetes/e2e"
+	"github.com/stretchr/testify/suite"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// istioUninjectTestingSuite is the entire Suite of tests for the "glooctl istio uninject" integration cases
+// NOTE: This suite is not intended to be run as a standalone test suite. It applies the "glooctl unistio inject" command
+// to an existing installation of Gloo Gateway where the istio-proxy and sds containers have already been injected
+// and verifies that the necessary resources are removed.
+type istioUninjectTestingSuite struct {
+	suite.Suite
+
+	ctx context.Context
+
+	// testInstallation contains all the metadata/utilities necessary to execute a series of tests
+	// against an installation of Gloo Gateway
+	testInstallation *e2e.TestInstallation
+}
+
+func NewIstioUninjectTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
+	return &istioUninjectTestingSuite{
+		ctx:              ctx,
+		testInstallation: testInst,
+	}
+}
+
+func (s *istioUninjectTestingSuite) TestCanUninject() {
+	// Uninject istio with glooctl. This assumes that the istio-proxy and sds containers have already been injected
+	// by the `glooctl istio inject` command in the `istioInjectTestingSuite`
+	out, err := s.testInstallation.Actions.Glooctl().IstioUninject(s.ctx, s.testInstallation.Metadata.InstallNamespace, s.testInstallation.ClusterContext.KubeContext)
+	s.Assert().NoError(err, "Failed to uninject istio")
+	s.Assert().Contains(out, "Istio was successfully uninjected")
+
+	matcher := gomega.And(
+		gomega.Not(matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.SDSContainerName})),
+		gomega.Not(matchers.PodMatches(matchers.ExpectedPod{ContainerName: istio.IstioProxyName})),
+	)
+	s.testInstallation.Assertions.EventuallyPodsMatches(s.ctx,
+		s.testInstallation.Metadata.InstallNamespace,
+		metav1.ListOptions{LabelSelector: fmt.Sprintf("gloo=%s", defaults.GatewayProxyName)},
+		matcher,
+	)
+}

test/kubernetes/e2e/features/istio/generate/generate.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
+	"github.com/solo-io/gloo/test/kubernetes/e2e/features/istio"
+	"github.com/solo-io/gloo/test/kubernetes/e2e/utils"
+	"github.com/solo-io/skv2/codegen/util"
+)
+
+const (
+	// exampleNs is the namespace where the resources will be created. Change this to the namespace where you want to create the resources
+	exampleNs = defaults.GlooSystem
+)
+
+// Dev tool to generate the manifest files for the test suite for demo and docs purposes
+//
+//go:generate go run ./generate.go
+func main() {
+	log.Println("starting generate for istio examples")
+
+	// use the Gloo Edge Gateway api resources with automtls enabled
+	edgeGatewayApiResources := istio.GetGlooGatewayEdgeResources(exampleNs)
+	automtlsGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("automtls-enabled-%s", istio.EdgeApisRoutingResourcesFileName))
+	err := utils.WriteResourcesToFile(edgeGatewayApiResources, automtlsGeneratedExample)
+	if err != nil {
+		panic(err)
+	}
+
+	log.Println("finished generate for istio examples")
+}

test/kubernetes/e2e/features/istio/generate/generated_example/automtls-enabled-edge-apis-routing.gen.yaml

@@ -0,0 +1,33 @@
+apiVersion: gateway.solo.io/v1
+kind: VirtualService
+metadata:
+  name: httpbin-vs
+  namespace: gloo-system
+spec:
+  virtualHost:
+    domains:
+    - httpbin
+    routes:
+    - matchers:
+      - prefix: /
+      routeAction:
+        single:
+          upstream:
+            name: httpbin-upstream
+            namespace: gloo-system
+status: {}
+---
+apiVersion: gloo.solo.io/v1
+kind: Upstream
+metadata:
+  name: httpbin-upstream
+  namespace: gloo-system
+spec:
+  kube:
+    selector:
+      app: httpbin
+    serviceName: httpbin
+    serviceNamespace: httpbin
+    servicePort: 8000
+status: {}
+---

test/kubernetes/e2e/features/istio/generate/generated_example/edge-apis-routing.gen.yaml

@@ -0,0 +1,33 @@
+apiVersion: gateway.solo.io/v1
+kind: VirtualService
+metadata:
+  name: httpbin-vs
+  namespace: gloo-system
+spec:
+  virtualHost:
+    domains:
+    - httpbin
+    routes:
+    - matchers:
+      - prefix: /
+      routeAction:
+        single:
+          upstream:
+            name: httpbin-upstream
+            namespace: gloo-system
+status: {}
+---
+apiVersion: gloo.solo.io/v1
+kind: Upstream
+metadata:
+  name: httpbin-upstream
+  namespace: gloo-system
+spec:
+  kube:
+    selector:
+      app: httpbin
+    serviceName: httpbin
+    serviceNamespace: httpbin
+    servicePort: 8000
+status: {}
+---