From 1fa9262c0b40c686401f629e72f097b1da3f566b Mon Sep 17 00:00:00 2001 From: Eric Hernandez Date: Wed, 5 Jun 2024 15:18:13 -0500 Subject: [PATCH] allows for image repos to be set via Environment Variables - useful when using a mirrored repo or no access to upstream repos --- clusterloader2/testing/load/config.yaml | 19 +++++++++++++++---- clusterloader2/testing/load/daemonset.yaml | 3 +-- clusterloader2/testing/load/deployment.yaml | 6 +++--- clusterloader2/testing/load/job.yaml | 2 +- .../load/modules/reconcile-objects.yaml | 15 ++++++++++++++- .../load/modules/scheduler-throughput.yaml | 3 +++ .../testing/load/simple-deployment.yaml | 3 +-- clusterloader2/testing/load/statefulset.yaml | 2 +- 8 files changed, 39 insertions(+), 14 deletions(-) diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml index 33e7ca5adf..a0d46aa492 100644 --- a/clusterloader2/testing/load/config.yaml +++ b/clusterloader2/testing/load/config.yaml @@ -74,7 +74,11 @@ {{$EXEC_TIMEOUT := DefaultParam .CL2_EXEC_TIMEOUT "3600s"}} {{$SLEEP_AFTER_EXEC_DURATION := DefaultParam .CL2_SLEEP_AFTER_EXEC_DURATION "0s"}} -{{$latencyPodImage := DefaultParam .CL2_LATENCY_POD_IMAGE "registry.k8s.io/pause:3.9"}} +## Image Repositories +{{$registryK8sRepo := DefaultParam .CL2_REGISTRY_K8S_REPO "registry.k8s.io"}} +{{$gcrRepo := DefaultParam .CL2_GCR_REPO "gcr.io"}} + +{{$latencyPodImage := DefaultParam .CL2_LATENCY_POD_IMAGE "pause:3.9"}} {{$defaultQps := DefaultParam .CL2_DEFAULT_QPS (IfThenElse (le .Nodes 500) 10 100)}} {{$ADDITIONAL_MEASUREMENT_MODULES := DefaultParam .CL2_ADDITIONAL_MEASUREMENT_MODULES nil}} @@ -188,7 +192,7 @@ steps: # so that when we measure pod startup latency, the image is already present on all nodes. # This way, the pod startup latency we measure excludes (or limits impact) of image pulling, # whuch matches our SLO definition: https://github.com/kubernetes/community/blob/master/sig-scalability/slos/pod_startup_latency.md. - daemonSetImage: {{$latencyPodImage}} + daemonSetImage: {{$registryK8sRepo}}/{{$latencyPodImage}} daemonSetEnv: "before update" daemonSetReplicas: 1 bigDeploymentSize: {{$BIG_GROUP_SIZE}} @@ -207,6 +211,8 @@ steps: mediumJobsPerNamespace: 1 smallJobSize: {{$SMALL_GROUP_SIZE}} smallJobsPerNamespace: 1 + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} @@ -227,6 +233,7 @@ steps: replicasPerNamespace: 1 schedulerThroughputNamespaces: {{$schedulerThroughputNamespaces}} schedulerThroughputPodsPerDeployment: {{$schedulerThroughputPodsPerDeployment}} + registryK8sRepo: {{$registryK8sRepo}} {{end}} - module: @@ -292,7 +299,7 @@ steps: params: namespaces: {{$namespaces}} minPodsInSmallCluster: {{$MIN_PODS_IN_SMALL_CLUSTERS}} - image: {{$latencyPodImage}} + image: {{$registryK8sRepo}}/{{$latencyPodImage}} {{end}} {{if $ENABLE_NETWORK_POLICY_ENFORCEMENT_LATENCY_TEST}} @@ -329,7 +336,7 @@ steps: {{end}} randomScaleFactor: {{$RANDOM_SCALE_FACTOR}} testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}} - daemonSetImage: {{$latencyPodImage}} + daemonSetImage: {{$registryK8sRepo}}/{{$latencyPodImage}} daemonSetEnv: "after update" daemonSetReplicas: 1 bigDeploymentSize: {{$BIG_GROUP_SIZE}} @@ -348,6 +355,8 @@ steps: mediumJobsPerNamespace: 1 smallJobSize: {{$SMALL_GROUP_SIZE}} smallJobsPerNamespace: 1 + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} @@ -390,6 +399,8 @@ steps: smallJobsPerNamespace: 0 pvSmallStatefulSetSize: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} pvMediumStatefulSetSize: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} diff --git a/clusterloader2/testing/load/daemonset.yaml b/clusterloader2/testing/load/daemonset.yaml index 5c69397693..d8bfff5898 100644 --- a/clusterloader2/testing/load/daemonset.yaml +++ b/clusterloader2/testing/load/daemonset.yaml @@ -1,5 +1,4 @@ {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} {{$Env := DefaultParam .Env ""}} {{$DaemonSetSurge := DefaultParam .CL2_DS_SURGE (MaxInt 10 (DivideInt .Nodes 20))}} # 5% of nodes, but not less than 10 {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} @@ -29,7 +28,7 @@ spec: hostNetwork: {{$HostNetworkMode}} containers: - name: {{.Name}} - image: {{$Image}} + image: {{.RegistryK8sRepo}}/pause:3.9 env: - name: TEST_ENV value: {{$Env}} diff --git a/clusterloader2/testing/load/deployment.yaml b/clusterloader2/testing/load/deployment.yaml index 02facca994..a7e1242bfa 100644 --- a/clusterloader2/testing/load/deployment.yaml +++ b/clusterloader2/testing/load/deployment.yaml @@ -45,13 +45,13 @@ spec: containers: {{if .EnableDNSTests}} {{if $USE_ADVANCED_DNSTEST}} - - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.4.0 + - image: {{.GcrRepo}}/k8s-staging-perf-tests/dnsperfgo:v1.4.0 ports: - containerPort: 9153 name: dnsperfmetrics protocol: TCP {{else}} - - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.2.0 + - image: {{.GcrRepo}}/k8s-staging-perf-tests/dnsperfgo:v1.2.0 {{end}} # Fetches the dns server from /etc/resolv.conf and # sends 1 query per second. @@ -72,7 +72,7 @@ spec: ports: - containerPort: 80 {{else}} - - image: registry.k8s.io/pause:3.9 + - image: {{.RegistryK8sRepo}}/pause:3.9 name: {{.Name}} {{end}} {{end}} diff --git a/clusterloader2/testing/load/job.yaml b/clusterloader2/testing/load/job.yaml index 27f27196a5..4c6af059aa 100644 --- a/clusterloader2/testing/load/job.yaml +++ b/clusterloader2/testing/load/job.yaml @@ -25,7 +25,7 @@ spec: containers: - name: {{.Name}} # TODO(#799): We should test the "run-to-completion" workflow and hence don't use pause pods. - image: registry.k8s.io/pause:3.9 + image: {{.RegistryK8sRepo}}/pause:3.9 resources: # Keep the CpuRequest/MemoryRequest request equal percentage of 1-core, 4GB node. # For now we're setting it to 0.5%. diff --git a/clusterloader2/testing/load/modules/reconcile-objects.yaml b/clusterloader2/testing/load/modules/reconcile-objects.yaml index 7bb244fbf5..12f3dca3b0 100644 --- a/clusterloader2/testing/load/modules/reconcile-objects.yaml +++ b/clusterloader2/testing/load/modules/reconcile-objects.yaml @@ -13,7 +13,7 @@ {{$operationTimeout := .operationTimeout}} # DaemonSets -{{$daemonSetImage := DefaultParam .daemonSetImage "registry.k8s.io/pause:3.9"}} +{{$daemonSetImage := DefaultParam .daemonSetImage "$registryK8sRepo/pause:3.9"}} {{$daemonSetReplicas := .daemonSetReplicas}} {{$daemonSetEnv := .daemonSetEnv}} @@ -55,6 +55,9 @@ {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE "enforcement-latency"}} {{$NET_POLICY_SERVER_EVERY_NTH_POD := DefaultParam .CL2_NET_POLICY_SERVER_EVERY_NTH_POD 3}} +{{$registryK8sRepo := .registryK8sRepo}} +{{$gcrRepo := .gcrRepo}} + steps: - name: Starting measurement for '{{$actionName}}' measurements: @@ -96,6 +99,7 @@ steps: templateFillMap: Image: {{$daemonSetImage}} Env: {{$daemonSetEnv}} + RegistryK8sRepo: {{$registryK8sRepo}} {{end}} - namespaceRange: min: 1 @@ -115,6 +119,7 @@ steps: ReplicasMin: {{MultiplyInt $bigDeploymentSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $bigDeploymentSize $maxReplicaFactor}} SvcName: big-service + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -132,6 +137,7 @@ steps: ReplicasMin: {{MultiplyInt $mediumDeploymentSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $mediumDeploymentSize $maxReplicaFactor}} SvcName: medium-service + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -154,6 +160,8 @@ steps: TargetLabelKey: {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_KEY}} TargetLabelValue: {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE}} NetPolServerOnEveryNthPod: {{$NET_POLICY_SERVER_EVERY_NTH_POD}} + RegistryK8sRepo: {{$registryK8sRepo}} + GcrRepo: {{$gcrRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -167,6 +175,7 @@ steps: templateFillMap: ReplicasMin: {{MultiplyInt $smallStatefulSetSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $smallStatefulSetSize $maxReplicaFactor}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -180,6 +189,7 @@ steps: templateFillMap: ReplicasMin: {{MultiplyInt $mediumStatefulSetSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $mediumStatefulSetSize $maxReplicaFactor}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -192,6 +202,7 @@ steps: Completions: {{MultiplyInt $smallJobSize $completionsFactor}} ReplicasMin: {{MultiplyInt $smallJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $smallJobSize $maxReplicaFactor}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -204,6 +215,7 @@ steps: Completions: {{MultiplyInt $mediumJobSize $completionsFactor}} ReplicasMin: {{MultiplyInt $mediumJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $mediumJobSize $maxReplicaFactor}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -216,6 +228,7 @@ steps: Completions: {{MultiplyInt $bigJobSize $completionsFactor}} ReplicasMin: {{MultiplyInt $bigJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $bigJobSize $maxReplicaFactor}} + RegistryK8sRepo: {{$registryK8sRepo}} {{if and $is_deleting $ENABLE_PVS}} - namespaceRange: min: 1 diff --git a/clusterloader2/testing/load/modules/scheduler-throughput.yaml b/clusterloader2/testing/load/modules/scheduler-throughput.yaml index 83c6296b27..d6ff89b96e 100644 --- a/clusterloader2/testing/load/modules/scheduler-throughput.yaml +++ b/clusterloader2/testing/load/modules/scheduler-throughput.yaml @@ -13,6 +13,8 @@ {{$SCHEDULER_THROUGHPUT_THRESHOLD := DefaultParam .CL2_SCHEDULER_THROUGHPUT_THRESHOLD 100}} {{$CHECK_IF_PODS_ARE_UPDATED := DefaultParam .CL2_CHECK_IF_PODS_ARE_UPDATED true}} +{{$registryK8sRepo := .registryK8sRepo}} + steps: {{if $is_creating}} - name: Creating scheduler throughput measurements @@ -54,6 +56,7 @@ steps: templateFillMap: Replicas: {{$schedulerThroughputPodsPerDeployment}} Group: scheduler-throughput + Image: {{$registryK8sRepo}}/pause:3.9 - name: Waiting for scheduler throughput pods to be {{$action}}d measurements: - Identifier: WaitForSchedulerThroughputDeployments diff --git a/clusterloader2/testing/load/simple-deployment.yaml b/clusterloader2/testing/load/simple-deployment.yaml index 92685b93e7..c19bac2124 100644 --- a/clusterloader2/testing/load/simple-deployment.yaml +++ b/clusterloader2/testing/load/simple-deployment.yaml @@ -4,7 +4,6 @@ {{$CpuRequest := DefaultParam .CpuRequest "5m"}} {{$EnvVar := DefaultParam .EnvVar "a"}} {{$MemoryRequest := DefaultParam .MemoryRequest "20M"}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} apiVersion: apps/v1 @@ -32,7 +31,7 @@ spec: - env: - name: ENV_VAR value: {{$EnvVar}} - image: {{$Image}} + image: {{.Image}} imagePullPolicy: IfNotPresent name: {{.Name}} ports: diff --git a/clusterloader2/testing/load/statefulset.yaml b/clusterloader2/testing/load/statefulset.yaml index 887b630573..788e8c451a 100644 --- a/clusterloader2/testing/load/statefulset.yaml +++ b/clusterloader2/testing/load/statefulset.yaml @@ -25,7 +25,7 @@ spec: hostNetwork: {{$HostNetworkMode}} containers: - name: {{.Name}} - image: registry.k8s.io/pause:3.9 + image: {{.RegistryK8sRepo}}/pause:3.9 ports: - containerPort: 80 name: web