Skip to content

Commit

Permalink
Make the parameters that limits the number of GameServers to add conf…
Browse files Browse the repository at this point in the history 
…igurable (#3950)

* Make gss controller parameters configurable

* lint

* gen install

* added maxPodPendingCount as parameter and also fixed spelling mistake with parallelism

* site-test

* install yaml update
  • Loading branch information
@vicentefb
vicentefb committed Aug 23, 2024
1 parent 2de0789 commit a78b18b
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 115 deletions.
189 changes: 110 additions & 79 deletions cmd/controller/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,34 +59,39 @@ import (
)

const (
enableStackdriverMetricsFlag = "stackdriver-exporter"
stackdriverLabels = "stackdriver-labels"
enablePrometheusMetricsFlag = "prometheus-exporter"
projectIDFlag = "gcp-project-id"
sidecarImageFlag = "sidecar-image"
sidecarCPURequestFlag = "sidecar-cpu-request"
sidecarCPULimitFlag = "sidecar-cpu-limit"
sidecarMemoryRequestFlag = "sidecar-memory-request"
sidecarMemoryLimitFlag = "sidecar-memory-limit"
sidecarRunAsUserFlag = "sidecar-run-as-user"
sdkServerAccountFlag = "sdk-service-account"
pullSidecarFlag = "always-pull-sidecar"
minPortFlag = "min-port"
maxPortFlag = "max-port"
additionalPortRangesFlag = "additional-port-ranges"
certFileFlag = "cert-file"
keyFileFlag = "key-file"
numWorkersFlag = "num-workers"
apiServerSustainedQPSFlag = "api-server-qps"
apiServerBurstQPSFlag = "api-server-qps-burst"
logDirFlag = "log-dir"
logLevelFlag = "log-level"
logSizeLimitMBFlag = "log-size-limit-mb"
kubeconfigFlag = "kubeconfig"
allocationBatchWaitTime = "allocation-batch-wait-time"
defaultResync = 30 * time.Second
podNamespace = "pod-namespace"
leaderElectionFlag = "leader-election"
enableStackdriverMetricsFlag = "stackdriver-exporter"
stackdriverLabels = "stackdriver-labels"
enablePrometheusMetricsFlag = "prometheus-exporter"
projectIDFlag = "gcp-project-id"
sidecarImageFlag = "sidecar-image"
sidecarCPURequestFlag = "sidecar-cpu-request"
sidecarCPULimitFlag = "sidecar-cpu-limit"
sidecarMemoryRequestFlag = "sidecar-memory-request"
sidecarMemoryLimitFlag = "sidecar-memory-limit"
sidecarRunAsUserFlag = "sidecar-run-as-user"
sdkServerAccountFlag = "sdk-service-account"
pullSidecarFlag = "always-pull-sidecar"
minPortFlag = "min-port"
maxPortFlag = "max-port"
additionalPortRangesFlag = "additional-port-ranges"
certFileFlag = "cert-file"
keyFileFlag = "key-file"
numWorkersFlag = "num-workers"
apiServerSustainedQPSFlag = "api-server-qps"
apiServerBurstQPSFlag = "api-server-qps-burst"
logDirFlag = "log-dir"
logLevelFlag = "log-level"
logSizeLimitMBFlag = "log-size-limit-mb"
kubeconfigFlag = "kubeconfig"
allocationBatchWaitTime = "allocation-batch-wait-time"
defaultResync = 30 * time.Second
podNamespace = "pod-namespace"
leaderElectionFlag = "leader-election"
maxCreationParallelismFlag = "max-creation-parallelism"
maxGameServerCreationsPerBatchFlag = "max-game-server-creations-per-batch"
maxDeletionParallelismFlag = "max-deletion-parallelism"
maxGameServerDeletionsPerBatchFlag = "max-game-server-deletions-per-batch"
maxPodPendingCountFlag = "max-pod-pending-count"
)

var (
Expand Down Expand Up @@ -218,7 +223,7 @@ func main() {
ctlConf.SidecarMemoryRequest, ctlConf.SidecarMemoryLimit, ctlConf.SidecarRunAsUser, ctlConf.SdkServiceAccount,
kubeClient, kubeInformerFactory, extClient, agonesClient, agonesInformerFactory)
gsSetController := gameserversets.NewController(health, gsCounter,
kubeClient, extClient, agonesClient, agonesInformerFactory)
kubeClient, extClient, agonesClient, agonesInformerFactory, ctlConf.MaxCreationParallelism, ctlConf.MaxDeletionParallelism, ctlConf.MaxGameServerCreationsPerBatch, ctlConf.MaxGameServerDeletionsPerBatch, ctlConf.MaxPodPendingCount)
fleetController := fleets.NewController(health, kubeClient, extClient, agonesClient, agonesInformerFactory)
fasController := fleetautoscalers.NewController(health,
kubeClient, extClient, agonesClient, agonesInformerFactory, gsCounter)
Expand Down Expand Up @@ -281,6 +286,12 @@ func parseEnvFlags() config {
viper.SetDefault(logLevelFlag, "Info")
viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks

viper.SetDefault(maxCreationParallelismFlag, 16)
viper.SetDefault(maxGameServerCreationsPerBatchFlag, 64)
viper.SetDefault(maxDeletionParallelismFlag, 64)
viper.SetDefault(maxGameServerDeletionsPerBatchFlag, 64)
viper.SetDefault(maxPodPendingCountFlag, 5000)

pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
pflag.String(sidecarCPURequestFlag, viper.GetString(sidecarCPURequestFlag), "Flag to overwrite the GameServer sidecar container's cpu request. Can also use SIDECAR_CPU_REQUEST env variable")
Expand All @@ -304,6 +315,11 @@ func parseEnvFlags() config {
pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
pflag.String(logDirFlag, viper.GetString(logDirFlag), "If set, store logs in a given directory.")
pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB")
pflag.Int32(maxCreationParallelismFlag, viper.GetInt32(maxCreationParallelismFlag), "Maximum number of parallelizing creation calls in GSS controller")
pflag.Int32(maxGameServerCreationsPerBatchFlag, viper.GetInt32(maxGameServerCreationsPerBatchFlag), "Maximum number of GameServer creation calls per batch")
pflag.Int32(maxDeletionParallelismFlag, viper.GetInt32(maxDeletionParallelismFlag), "Maximum number of parallelizing deletion calls in GSS controller")
pflag.Int32(maxGameServerDeletionsPerBatchFlag, viper.GetInt32(maxGameServerDeletionsPerBatchFlag), "Maximum number of GameServers deletion calls per batch")
pflag.Int32(maxPodPendingCountFlag, viper.GetInt32(maxPodPendingCountFlag), "Maximum number of pending pods per game server set")
pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level")
pflag.Duration(allocationBatchWaitTime, viper.GetDuration(allocationBatchWaitTime), "Flag to configure the waiting period between allocations batches")
pflag.String(podNamespace, viper.GetString(podNamespace), "namespace of current pod")
Expand Down Expand Up @@ -337,6 +353,11 @@ func parseEnvFlags() config {
runtime.Must(viper.BindEnv(logLevelFlag))
runtime.Must(viper.BindEnv(logDirFlag))
runtime.Must(viper.BindEnv(logSizeLimitMBFlag))
runtime.Must(viper.BindEnv(maxCreationParallelismFlag))
runtime.Must(viper.BindEnv(maxGameServerCreationsPerBatchFlag))
runtime.Must(viper.BindEnv(maxDeletionParallelismFlag))
runtime.Must(viper.BindEnv(maxGameServerDeletionsPerBatchFlag))
runtime.Must(viper.BindEnv(maxPodPendingCountFlag))
runtime.Must(viper.BindEnv(allocationBatchWaitTime))
runtime.Must(viper.BindEnv(podNamespace))
runtime.Must(viper.BindEnv(leaderElectionFlag))
Expand Down Expand Up @@ -376,31 +397,36 @@ func parseEnvFlags() config {
}

return config{
PortRanges: portRanges,
SidecarImage: viper.GetString(sidecarImageFlag),
SidecarCPURequest: requestCPU,
SidecarCPULimit: limitCPU,
SidecarMemoryRequest: requestMemory,
SidecarMemoryLimit: limitMemory,
SidecarRunAsUser: int(viper.GetInt32(sidecarRunAsUserFlag)),
SdkServiceAccount: viper.GetString(sdkServerAccountFlag),
AlwaysPullSidecar: viper.GetBool(pullSidecarFlag),
KeyFile: viper.GetString(keyFileFlag),
CertFile: viper.GetString(certFileFlag),
KubeConfig: viper.GetString(kubeconfigFlag),
PrometheusMetrics: viper.GetBool(enablePrometheusMetricsFlag),
Stackdriver: viper.GetBool(enableStackdriverMetricsFlag),
GCPProjectID: viper.GetString(projectIDFlag),
NumWorkers: int(viper.GetInt32(numWorkersFlag)),
APIServerSustainedQPS: int(viper.GetInt32(apiServerSustainedQPSFlag)),
APIServerBurstQPS: int(viper.GetInt32(apiServerBurstQPSFlag)),
LogDir: viper.GetString(logDirFlag),
LogLevel: viper.GetString(logLevelFlag),
LogSizeLimitMB: int(viper.GetInt32(logSizeLimitMBFlag)),
StackdriverLabels: viper.GetString(stackdriverLabels),
AllocationBatchWaitTime: viper.GetDuration(allocationBatchWaitTime),
PodNamespace: viper.GetString(podNamespace),
LeaderElection: viper.GetBool(leaderElectionFlag),
PortRanges: portRanges,
SidecarImage: viper.GetString(sidecarImageFlag),
SidecarCPURequest: requestCPU,
SidecarCPULimit: limitCPU,
SidecarMemoryRequest: requestMemory,
SidecarMemoryLimit: limitMemory,
SidecarRunAsUser: int(viper.GetInt32(sidecarRunAsUserFlag)),
SdkServiceAccount: viper.GetString(sdkServerAccountFlag),
AlwaysPullSidecar: viper.GetBool(pullSidecarFlag),
KeyFile: viper.GetString(keyFileFlag),
CertFile: viper.GetString(certFileFlag),
KubeConfig: viper.GetString(kubeconfigFlag),
PrometheusMetrics: viper.GetBool(enablePrometheusMetricsFlag),
Stackdriver: viper.GetBool(enableStackdriverMetricsFlag),
GCPProjectID: viper.GetString(projectIDFlag),
NumWorkers: int(viper.GetInt32(numWorkersFlag)),
APIServerSustainedQPS: int(viper.GetInt32(apiServerSustainedQPSFlag)),
APIServerBurstQPS: int(viper.GetInt32(apiServerBurstQPSFlag)),
LogDir: viper.GetString(logDirFlag),
LogLevel: viper.GetString(logLevelFlag),
LogSizeLimitMB: int(viper.GetInt32(logSizeLimitMBFlag)),
MaxGameServerCreationsPerBatch: int(viper.GetInt32(maxGameServerCreationsPerBatchFlag)),
MaxCreationParallelism: int(viper.GetInt32(maxCreationParallelismFlag)),
MaxGameServerDeletionsPerBatch: int(viper.GetInt32(maxGameServerDeletionsPerBatchFlag)),
MaxDeletionParallelism: int(viper.GetInt32(maxDeletionParallelismFlag)),
MaxPodPendingCount: int(viper.GetInt32(maxPodPendingCountFlag)),
StackdriverLabels: viper.GetString(stackdriverLabels),
AllocationBatchWaitTime: viper.GetDuration(allocationBatchWaitTime),
PodNamespace: viper.GetString(podNamespace),
LeaderElection: viper.GetBool(leaderElectionFlag),
}
}

Expand Down Expand Up @@ -429,31 +455,36 @@ func parsePortRanges(s string) (map[string]portallocator.PortRange, error) {

// config stores all required configuration to create a game server controller.
type config struct {
PortRanges map[string]portallocator.PortRange
SidecarImage string
SidecarCPURequest resource.Quantity
SidecarCPULimit resource.Quantity
SidecarMemoryRequest resource.Quantity
SidecarMemoryLimit resource.Quantity
SidecarRunAsUser int
SdkServiceAccount string
AlwaysPullSidecar bool
PrometheusMetrics bool
Stackdriver bool
StackdriverLabels string
KeyFile string
CertFile string
KubeConfig string
GCPProjectID string
NumWorkers int
APIServerSustainedQPS int
APIServerBurstQPS int
LogDir string
LogLevel string
LogSizeLimitMB int
AllocationBatchWaitTime time.Duration
PodNamespace string
LeaderElection bool
PortRanges map[string]portallocator.PortRange
SidecarImage string
SidecarCPURequest resource.Quantity
SidecarCPULimit resource.Quantity
SidecarMemoryRequest resource.Quantity
SidecarMemoryLimit resource.Quantity
SidecarRunAsUser int
SdkServiceAccount string
AlwaysPullSidecar bool
PrometheusMetrics bool
Stackdriver bool
StackdriverLabels string
KeyFile string
CertFile string
KubeConfig string
GCPProjectID string
NumWorkers int
APIServerSustainedQPS int
APIServerBurstQPS int
LogDir string
LogLevel string
LogSizeLimitMB int
MaxGameServerCreationsPerBatch int
MaxCreationParallelism int
MaxGameServerDeletionsPerBatch int
MaxDeletionParallelism int
MaxPodPendingCount int
AllocationBatchWaitTime time.Duration
PodNamespace string
LeaderElection bool
}

// validate ensures the ctlConfig data is valid.
Expand Down
10 changes: 10 additions & 0 deletions install/helm/agones/templates/controller.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,16 @@ spec:
value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
- name: NUM_WORKERS
value: {{ .Values.agones.controller.numWorkers | quote }}
- name: MAX_CREATION_PARALLELISM
value: {{ .Values.agones.controller.maxCreationParallelism | quote }}
- name: MAX_GAME_SERVER_CREATIONS_PER_BATCH
value: {{ .Values.agones.controller.maxGameServerCreationsPerBatch | quote }}
- name: MAX_DELETION_PARALLELISM
value: {{ .Values.agones.controller.maxDeletionParallelism | quote }}
- name: MAX_GAME_SERVER_DELETIONS_PER_BATCH
value: {{ .Values.agones.controller.maxGameServerDeletionsPerBatch | quote }}
- name: MAX_POD_PENDING_COUNT
value: {{ .Values.agones.controller.maxPodPendingCount | quote }}
- name: API_SERVER_QPS
value: {{ .Values.agones.controller.apiServerQPS | quote }}
- name: API_SERVER_QPS_BURST
Expand Down
5 changes: 5 additions & 0 deletions install/helm/agones/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ agones:
numWorkers: 100
apiServerQPS: 400
apiServerQPSBurst: 500
maxCreationParallelism: 16
maxGameServerCreationsPerBatch: 64
maxDeletionParallelism: 64
maxGameServerDeletionsPerBatch: 64
maxPodPendingCount: 5000
http:
port: 8080
healthCheck:
Expand Down
10 changes: 10 additions & 0 deletions install/yaml/install.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17524,6 +17524,16 @@ spec:
value: ""
- name: NUM_WORKERS
value: "100"
- name: MAX_CREATION_PARALLELISM
value: "16"
- name: MAX_GAME_SERVER_CREATIONS_PER_BATCH
value: "64"
- name: MAX_DELETION_PARALLELISM
value: "64"
- name: MAX_GAME_SERVER_DELETIONS_PER_BATCH
value: "64"
- name: MAX_POD_PENDING_COUNT
value: "5000"
- name: API_SERVER_QPS
value: "400"
- name: API_SERVER_QPS_BURST
Expand Down
Loading

0 comments on commit a78b18b

Please sign in to comment.