diff --git a/changelog/v1.15.0-beta16/performance-tests-type-error.yaml b/changelog/v1.15.0-beta16/performance-tests-type-error.yaml new file mode 100644 index 00000000000..14e2f6ea8e7 --- /dev/null +++ b/changelog/v1.15.0-beta16/performance-tests-type-error.yaml @@ -0,0 +1,22 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/5116 + resolvesIssue: false + description: >- + Resolve a bug introduced in #8415 due to type mismatch in the translation performance test table's descriptor func + skipCI-kube-tests:true + skipCI-docs-build:true + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/5116 + resolvesIssue: false + description: >- + Resolve a bug introduced in #8415 due to 0-indexing names of some resources but not all + skipCI-kube-tests:true + skipCI-docs-build:true + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/4918 + resolvesIssue: false + description: >- + Add a Darwin implementation of benchmark measurement to allow tests to run locally + skipCI-kube-tests:true + skipCI-docs-build:true \ No newline at end of file diff --git a/projects/gloo/pkg/translator/performance_linux_test.go b/projects/gloo/pkg/translator/performance_test.go similarity index 92% rename from projects/gloo/pkg/translator/performance_linux_test.go rename to projects/gloo/pkg/translator/performance_test.go index 8fa553f0afc..7db470affff 100644 --- a/projects/gloo/pkg/translator/performance_linux_test.go +++ b/projects/gloo/pkg/translator/performance_test.go @@ -7,8 +7,6 @@ import ( "github.com/solo-io/go-utils/contextutils" - "github.com/solo-io/gloo/test/testutils" - validationutils "github.com/solo-io/gloo/projects/gloo/pkg/utils/validation" "github.com/solo-io/gloo/test/ginkgo/labels" @@ -47,9 +45,6 @@ type benchmarkConfig struct { benchmarkMatchers []types.GomegaMatcher // matchers representing the assertions we wish to make for a particular entry } -// These tests only compile and run on Linux machines due to the use of the go-utils benchmarking package which is only -// compatible with Linux - // Tests are run as part of the "Nightly" action in a GHA using the default Linux runner // More info on that machine can be found here: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources // When developing new tests, users should manually run that action in order to test performance under the same parameters @@ -62,8 +57,6 @@ var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Perfor ) BeforeEach(func() { - testutils.LinuxOnly("uses go-utils benchmarking.Measure() which only compiles on Linux") - ctrl = gomock.NewController(T) settings = &v1.Settings{} @@ -110,7 +103,7 @@ var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Perfor Expect(apiSnap.Proxies).NotTo(BeEmpty()) proxy = apiSnap.Proxies[0] - desc := generateDesc(apiSnap, config, labels...) + desc := generateDesc(snapBuilder, config, labels...) experiment := gmeasure.NewExperiment(fmt.Sprintf("Experiment - %s", desc)) @@ -152,7 +145,7 @@ var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Perfor Expect(durations).Should(And(config.benchmarkMatchers...)) }, generateDesc, // generate descriptions for table entries with nil descriptions - Entry("basic", basicSnap, basicConfig), + Entry("basic", gloohelpers.NewInjectedSnapshotBuilder(basicSnap), basicConfig), Entry(nil, gloohelpers.NewScaledSnapshotBuilder().WithUpstreamCount(10).WithEndpointCount(1), basicConfig, "upstream scale"), Entry(nil, gloohelpers.NewScaledSnapshotBuilder().WithUpstreamCount(1000).WithEndpointCount(1), oneKUpstreamsConfig, "upstream scale"), Entry(nil, gloohelpers.NewScaledSnapshotBuilder().WithUpstreamCount(1).WithEndpointCount(10), basicConfig, "endpoint scale"), @@ -165,14 +158,18 @@ var _ = Describe("Translation - Benchmarking Tests", Serial, Label(labels.Perfor ) }) -func generateDesc(apiSnap *v1snap.ApiSnapshot, _ benchmarkConfig, labels ...string) string { +func generateDesc(b *gloohelpers.ScaledSnapshotBuilder, _ benchmarkConfig, labels ...string) string { labelPrefix := "" if len(labels) > 0 { labelPrefix = fmt.Sprintf("(%s) ", strings.Join(labels, ", ")) } + if b.HasInjectedSnapshot() { + return fmt.Sprintf("%sinjected snapshot", labelPrefix) + } + // If/when additional Snapshot fields are included in testing, the description should be updated accordingly - return fmt.Sprintf("%s%d endpoint(s), %d upstream(s)", labelPrefix, len(apiSnap.Endpoints), len(apiSnap.Upstreams)) + return fmt.Sprintf("%s%d endpoint(s), %d upstream(s)", labelPrefix, b.EndpointCount(), b.UpstreamCount()) } // Test assets: Add blocks for logical groupings of tests, including: @@ -188,8 +185,8 @@ var basicSnap = &v1snap.ApiSnapshot{ }, }, }, - Endpoints: []*v1.Endpoint{gloohelpers.Endpoint(1)}, - Upstreams: []*v1.Upstream{gloohelpers.Upstream(1)}, + Endpoints: []*v1.Endpoint{gloohelpers.Endpoint(0)}, + Upstreams: []*v1.Upstream{gloohelpers.Upstream(0)}, } var basicConfig = benchmarkConfig{ diff --git a/test/helpers/benchmark.go b/test/helpers/benchmark.go new file mode 100644 index 00000000000..71632373d65 --- /dev/null +++ b/test/helpers/benchmark.go @@ -0,0 +1,13 @@ +package helpers + +import "time" + +// Result represents the result of measuring a function's execution time. +type Result struct { + // Time spent in user mode + Utime time.Duration + // Time spent in kernel mode + Stime time.Duration + // Time spent in user mode + kernel mode + Total time.Duration +} diff --git a/test/helpers/benchmark_darwin.go b/test/helpers/benchmark_darwin.go new file mode 100644 index 00000000000..779a08b11d1 --- /dev/null +++ b/test/helpers/benchmark_darwin.go @@ -0,0 +1,17 @@ +package helpers + +import ( + "time" +) + +// MeasureIgnore0ns as implemented here for Mac/Darwin is meant to be used in performance tests when developing locally +// It is a less-precise method for measuring than the Linux implementation, and targets should be derived based on +// performance when running on the Linux GHA runner we use for Nightly tests +func MeasureIgnore0ns(f func()) (Result, bool, error) { + before := time.Now() + f() + elapsed := time.Since(before) + return Result{ + Total: elapsed, + }, false, nil +} diff --git a/test/helpers/scaled_snapshots.go b/test/helpers/scaled_snapshots.go index 2800266d9f1..5eba9863956 100644 --- a/test/helpers/scaled_snapshots.go +++ b/test/helpers/scaled_snapshots.go @@ -20,6 +20,8 @@ import ( // contains a builder for each sub-resource type which is responsible for building instances of that resource // Additional fields should be added as needed type ScaledSnapshotBuilder struct { + injectedSnap *gloosnapshot.ApiSnapshot + epCount int usCount int @@ -34,6 +36,14 @@ func NewScaledSnapshotBuilder() *ScaledSnapshotBuilder { } } +// NewInjectedSnapshotBuilder takes a snapshot object to be returned directly by Build() +// All other settings on a builder with an InjectedSnapshot will be ignored +func NewInjectedSnapshotBuilder(snap *gloosnapshot.ApiSnapshot) *ScaledSnapshotBuilder { + return &ScaledSnapshotBuilder{ + injectedSnap: snap, + } +} + func (b *ScaledSnapshotBuilder) WithUpstreamCount(n int) *ScaledSnapshotBuilder { b.usCount = n return b @@ -54,9 +64,27 @@ func (b *ScaledSnapshotBuilder) WithEndpointBuilder(eb *EndpointBuilder) *Scaled return b } +/* Getter funcs to be used by the description generator */ + +func (b *ScaledSnapshotBuilder) HasInjectedSnapshot() bool { + return b.injectedSnap != nil +} + +func (b *ScaledSnapshotBuilder) UpstreamCount() int { + return b.usCount +} + +func (b *ScaledSnapshotBuilder) EndpointCount() int { + return b.epCount +} + // Build generates a snapshot populated with the specified number of each resource for the builder, using the // sub-resource builders to build each sub-resource func (b *ScaledSnapshotBuilder) Build() *gloosnapshot.ApiSnapshot { + if b.injectedSnap != nil { + return b.injectedSnap + } + endpointList := make(v1.EndpointList, b.epCount) for i := 0; i < b.epCount; i++ { endpointList[i] = b.epBuilder.Build(i) @@ -142,7 +170,7 @@ func route(i int) *v1.Route { func routes(n int) []*v1.Route { routes := make([]*v1.Route, n) for i := 0; i < n; i++ { - routes[i] = route(i + 1) // names are 1-indexed + routes[i] = route(i) } return routes } @@ -182,8 +210,8 @@ func tcpListener() *v1.Listener { Single: &v1.Destination{ DestinationType: &v1.Destination_Upstream{ Upstream: &core.ResourceRef{ - Name: upMeta(1).GetName(), - Namespace: upMeta(1).GetNamespace(), + Name: upMeta(0).GetName(), + Namespace: upMeta(0).GetNamespace(), }, }, }, diff --git a/test/helpers/scaled_snapshots_test.go b/test/helpers/scaled_snapshots_test.go index 4f87f2b06a6..5e5489dcb31 100644 --- a/test/helpers/scaled_snapshots_test.go +++ b/test/helpers/scaled_snapshots_test.go @@ -3,7 +3,10 @@ package helpers_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" + "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/gloosnapshot" "github.com/solo-io/gloo/test/helpers" + "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" ) var _ = Describe("ScaledSnapshotBuilder", func() { @@ -54,3 +57,20 @@ var _ = Describe("ScaledSnapshotBuilder", func() { }) }) }) + +var _ = Describe("InjectedSnapshotBuilder", func() { + It("returns the injected snapshot regardless of other settings", func() { + inSnap := &gloosnapshot.ApiSnapshot{ + Upstreams: []*v1.Upstream{ + { + Metadata: &core.Metadata{ + Name: "injected-name", + Namespace: "injected-namespace", + }, + }, + }, + } + outSnap := helpers.NewInjectedSnapshotBuilder(inSnap).WithUpstreamCount(10).Build() + Expect(outSnap).To(Equal(inSnap)) + }) +})