feat: Add option to configure extra pod capacity for alternate cnis

pkg/operator/options/options.go

@@ -44,6 +44,7 @@ type Options struct {
 	VMMemoryOverheadPercent float64
 	InterruptionQueue       string
 	ReservedENIs            int
+	MaxPodsExtraCapacity    int
 }
 
 func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
@@ -56,6 +57,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) {
 	fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", utils.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.")
 	fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is the name of the SQS queue used for processing interruption events from EC2. Interruption handling is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.")
 	fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.")
+	fs.IntVar(&o.MaxPodsExtraCapacity, "max-pods-extra-capacity", env.WithDefaultInt("MAX_PODS_EXTRA_CAPACITY", 2), "Extra pod capacity allocatable on nodes due to host networked pods that do not consume ENIs. The default is for aws-cni and kube-proxy pods, which are not included in the calculations for max-pods or kube-reserved.")
 }
 
 func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error {

pkg/operator/options/options_validation.go

@@ -28,6 +28,7 @@ func (o Options) Validate() error {
 		o.validateVMMemoryOverheadPercent(),
 		o.validateAssumeRoleDuration(),
 		o.validateReservedENIs(),
+		o.validateMaxPodsExtraCapacity(),
 		o.validateRequiredFields(),
 	)
 }
@@ -66,6 +67,13 @@ func (o Options) validateReservedENIs() error {
 	return nil
 }
 
+func (o Options) validateMaxPodsExtraCapacity() error {
+	if o.MaxPodsExtraCapacity < 0 {
+		return fmt.Errorf("max-pods-extra-capacity cannot be negative")
+	}
+	return nil
+}
+
 func (o Options) validateRequiredFields() error {
 	if o.ClusterName == "" {
 		return fmt.Errorf("missing field, cluster-name")

pkg/operator/options/suite_test.go

@@ -65,7 +65,8 @@ var _ = Describe("Options", func() {
 			"--isolated-vpc",
 			"--vm-memory-overhead-percent", "0.1",
 			"--interruption-queue", "env-cluster",
-			"--reserved-enis", "10")
+			"--reserved-enis", "10",
+			"--max-pods-extra-capacity", "1")
 		Expect(err).ToNot(HaveOccurred())
 		expectOptionsEqual(opts, test.Options(test.OptionsFields{
 			AssumeRoleARN:           lo.ToPtr("env-role"),
@@ -77,6 +78,7 @@ var _ = Describe("Options", func() {
 			VMMemoryOverheadPercent: lo.ToPtr[float64](0.1),
 			InterruptionQueue:       lo.ToPtr("env-cluster"),
 			ReservedENIs:            lo.ToPtr(10),
+			MaxPodsExtraCapacity:    lo.ToPtr(1),
 		}))
 	})
 	It("should correctly fallback to env vars when CLI flags aren't set", func() {
@@ -89,6 +91,7 @@ var _ = Describe("Options", func() {
 		os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1")
 		os.Setenv("INTERRUPTION_QUEUE", "env-cluster")
 		os.Setenv("RESERVED_ENIS", "10")
+		os.Setenv("MAX_PODS_EXTRA_CAPACITY", "1")
 
 		// Add flags after we set the environment variables so that the parsing logic correctly refers
 		// to the new environment variable values
@@ -105,6 +108,7 @@ var _ = Describe("Options", func() {
 			VMMemoryOverheadPercent: lo.ToPtr[float64](0.1),
 			InterruptionQueue:       lo.ToPtr("env-cluster"),
 			ReservedENIs:            lo.ToPtr(10),
+			MaxPodsExtraCapacity:    lo.ToPtr(1),
 		}))
 	})
 
@@ -132,6 +136,10 @@ var _ = Describe("Options", func() {
 			err := opts.Parse(fs, "--cluster-name", "test-cluster", "--reserved-enis", "-1")
 			Expect(err).To(HaveOccurred())
 		})
+		It("should fail when maxPodsExtraCapacity is negative", func() {
+			err := opts.Parse(fs, "--cluster-name", "test-cluster", "--max-pods-extra-capacity", "-1")
+			Expect(err).To(HaveOccurred())
+		})
 	})
 })
 
@@ -146,4 +154,5 @@ func expectOptionsEqual(optsA *options.Options, optsB *options.Options) {
 	Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent))
 	Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue))
 	Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs))
+	Expect(optsA.MaxPodsExtraCapacity).To(Equal(optsB.MaxPodsExtraCapacity))
 }

pkg/providers/instancetype/types.go

@@ -348,7 +348,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q
 		return resource.NewQuantity(0, resource.DecimalSI)
 	}
 	addressesPerInterface := *info.NetworkInfo.Ipv4AddressesPerInterface
-	return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + 2))
+	return resources.Quantity(fmt.Sprint(usableNetworkInterfaces*(addressesPerInterface-1) + int64(options.FromContext(ctx).MaxPodsExtraCapacity)))
 }
 
 func privateIPv4Address(instanceTypeName string) *resource.Quantity {

pkg/test/options.go

@@ -34,6 +34,7 @@ type OptionsFields struct {
 	VMMemoryOverheadPercent *float64
 	InterruptionQueue       *string
 	ReservedENIs            *int
+	MaxPodsExtraCapacity    *int
 }
 
 func Options(overrides ...OptionsFields) *options.Options {
@@ -53,5 +54,6 @@ func Options(overrides ...OptionsFields) *options.Options {
 		VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075),
 		InterruptionQueue:       lo.FromPtrOr(opts.InterruptionQueue, ""),
 		ReservedENIs:            lo.FromPtrOr(opts.ReservedENIs, 0),
+		MaxPodsExtraCapacity:    lo.FromPtrOr(opts.MaxPodsExtraCapacity, 2),
 	}
 }