From 77aa7e21cdf74f2a18e44f12d62e8312b1c772b2 Mon Sep 17 00:00:00 2001 From: tan90github <81003453+tan90github@users.noreply.github.com> Date: Wed, 30 Aug 2023 16:22:49 +0800 Subject: [PATCH] koord-scheduler: export koordinator-root-quota as CRD object (#1555) Signed-off-by: wangyang55 --- apis/extension/elastic_quota.go | 2 +- .../elasticquota/core/group_quota_manager.go | 25 +++- .../core/group_quota_manager_test.go | 135 ++++++++++++++++-- pkg/scheduler/plugins/elasticquota/plugin.go | 1 + .../plugins/elasticquota/plugin_helper.go | 27 ++++ .../plugins/elasticquota/plugin_test.go | 2 +- pkg/webhook/elasticquota/quota_topology.go | 2 +- .../elasticquota/quota_topology_check.go | 3 + 8 files changed, 184 insertions(+), 13 deletions(-) diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go index d1fd292067..4835e017fd 100644 --- a/apis/extension/elastic_quota.go +++ b/apis/extension/elastic_quota.go @@ -42,7 +42,7 @@ const ( func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string { parentName := quota.Labels[LabelQuotaParent] - if parentName == "" { + if parentName == "" && quota.Name != RootQuotaName { return RootQuotaName //default return RootQuotaName } return parentName diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go index db71fb9aac..faae497b40 100644 --- a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go @@ -77,6 +77,7 @@ func NewGroupQuotaManager(systemGroupMax, defaultGroupMax v1.ResourceList, nodeL quotaManager.quotaInfoMap[extension.SystemQuotaName].setMaxQuotaNoLock(systemGroupMax) quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName) quotaManager.quotaInfoMap[extension.DefaultQuotaName].setMaxQuotaNoLock(defaultGroupMax) + quotaManager.quotaInfoMap[extension.RootQuotaName] = NewQuotaInfo(true, false, extension.RootQuotaName, "") quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName) quotaManager.setScaleMinQuotaEnabled(true) return quotaManager @@ -148,7 +149,7 @@ func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq curQuotaInfo := curToAllParInfos[i] oldSubLimitReq := curQuotaInfo.getLimitRequestNoLock() curQuotaInfo.addRequestNonNegativeNoLock(deltaReq) - if curQuotaInfo.Name == extension.SystemQuotaName || curQuotaInfo.Name == extension.DefaultQuotaName { + if curQuotaInfo.Name == extension.RootQuotaName { return } newSubLimitReq := curQuotaInfo.getLimitRequestNoLock() @@ -199,6 +200,10 @@ func (gqm *GroupQuotaManager) RefreshRuntimeNoLock(quotaName string) v1.Resource return nil } + if quotaName == extension.RootQuotaName { + return gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy() + } + if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName { return quotaInfo.getMax() } @@ -210,6 +215,9 @@ func (gqm *GroupQuotaManager) RefreshRuntimeNoLock(quotaName string) v1.Resource totalRes := gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy() for i := len(curToAllParInfos) - 1; i >= 0; i-- { quotaInfo = curToAllParInfos[i] + if quotaInfo.Name == extension.RootQuotaName { + continue + } parRuntimeQuotaCalculator := gqm.getRuntimeQuotaCalculatorByNameNoLock(quotaInfo.ParentName) if parRuntimeQuotaCalculator == nil { klog.Errorf("treeWrapper not exist! parentQuotaName:%v", quotaInfo.ParentName) @@ -265,7 +273,7 @@ func (gqm *GroupQuotaManager) getCurToAllParentGroupQuotaInfoNoLock(quotaName st for true { curToAllParInfos = append(curToAllParInfos, quotaInfo) - if quotaInfo.ParentName == extension.RootQuotaName { + if quotaInfo.Name == extension.RootQuotaName { break } @@ -381,6 +389,7 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() { childRequestMap, childUsedMap := make(quotaResMapType), make(quotaResMapType) for quotaName, topoNode := range gqm.quotaTopoNodeMap { if quotaName == extension.RootQuotaName { + gqm.resetRootQuotaUsedAndRequest() continue } topoNode.quotaInfo.lock.Lock() @@ -791,3 +800,15 @@ func (gqm *GroupQuotaManager) ResyncNodes() { // TODO: remove none-exist node } + +func (gqm *GroupQuotaManager) resetRootQuotaUsedAndRequest() { + rootQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName) + rootQuotaInfo.lock.Lock() + defer rootQuotaInfo.lock.Unlock() + + systemQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.SystemQuotaName) + defaultQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.DefaultQuotaName) + + rootQuotaInfo.CalculateInfo.Used = quotav1.Add(systemQuotaInfo.GetUsed(), defaultQuotaInfo.GetUsed()) + rootQuotaInfo.CalculateInfo.Request = quotav1.Add(systemQuotaInfo.GetRequest(), defaultQuotaInfo.GetRequest()) +} diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go index 75a0bf5487..912e90b5d6 100644 --- a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go +++ b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go @@ -54,7 +54,7 @@ func TestGroupQuotaManager_QuotaAdd(t *testing.T) { } assert.Equal(t, 6, len(gqm.runtimeQuotaCalculatorMap)) - assert.Equal(t, 7, len(gqm.quotaInfoMap)) + assert.Equal(t, 8, len(gqm.quotaInfoMap)) assert.Equal(t, 2, len(gqm.resourceKeys)) } @@ -143,7 +143,7 @@ func TestGroupQuotaManager_UpdateQuota(t *testing.T) { gqm := NewGroupQuotaManagerForTest() quota := CreateQuota("test1", "test-parent", 64, 100*GigaByte, 50, 80*GigaByte, true, false) gqm.UpdateQuota(quota, false) - assert.Equal(t, len(gqm.quotaInfoMap), 3) + assert.Equal(t, len(gqm.quotaInfoMap), 4) } func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) { @@ -180,7 +180,7 @@ func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) { quota2 := AddQuotaToManager(t, gqm, "test2", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, false) quota3 := AddQuotaToManager(t, gqm, "test3", extension.RootQuotaName, 96, 160*GigaByte, 40, 40*GigaByte, true, false) assert.Equal(t, 4, len(gqm.runtimeQuotaCalculatorMap)) - assert.Equal(t, 5, len(gqm.quotaInfoMap)) + assert.Equal(t, 6, len(gqm.quotaInfoMap)) err := gqm.UpdateQuota(quota1, true) assert.Nil(t, err) @@ -198,13 +198,13 @@ func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) { assert.True(t, quotaInfo == nil) assert.Equal(t, 1, len(gqm.runtimeQuotaCalculatorMap)) - assert.Equal(t, 2, len(gqm.quotaInfoMap)) + assert.Equal(t, 3, len(gqm.quotaInfoMap)) AddQuotaToManager(t, gqm, "youku", extension.RootQuotaName, 96, 160*GigaByte, 70, 70*GigaByte, true, false) quotaInfo = gqm.GetQuotaInfoByName("youku") assert.NotNil(t, quotaInfo) assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap)) - assert.Equal(t, 3, len(gqm.quotaInfoMap)) + assert.Equal(t, 4, len(gqm.quotaInfoMap)) } func TestGroupQuotaManager_UpdateQuotaDeltaRequest(t *testing.T) { @@ -325,7 +325,7 @@ func TestGroupQuotaManager_MultiQuotaAdd(t *testing.T) { AddQuotaToManager(t, gqm, "test2-sub2-1", "test2-sub2", 96, 160*GigaByte, 0, 0*GigaByte, true, false) assert.Equal(t, 9, len(gqm.runtimeQuotaCalculatorMap)) - assert.Equal(t, 10, len(gqm.quotaInfoMap)) + assert.Equal(t, 11, len(gqm.quotaInfoMap)) assert.Equal(t, 2, len(gqm.resourceKeys)) assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].quotaTree[v1.ResourceCPU].quotaNodes)) assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap["test2"].quotaTree[v1.ResourceCPU].quotaNodes)) @@ -1088,8 +1088,9 @@ func NewGroupQuotaManagerForTest() *GroupQuotaManager { scaleMinQuotaManager: NewScaleMinQuotaManager(), quotaTopoNodeMap: make(map[string]*QuotaTopoNode), } - quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, "") - quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, "") + quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, extension.RootQuotaName) + quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName) + quotaManager.quotaInfoMap[extension.RootQuotaName] = NewQuotaInfo(true, false, extension.RootQuotaName, "") quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName) return quotaManager } @@ -1305,3 +1306,121 @@ func TestGroupQuotaManager_IsParent(t *testing.T) { assert.False(t, qi2Info.IsParent) } + +func TestGroupQuotaManager_UpdateRootQuotaUsed(t *testing.T) { + expectedTotalUsed := createResourceList(0, 0) + gqm := NewGroupQuotaManagerForTest() + gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) + + sysUsed := createResourceList(10, 30) + expectedTotalUsed = quotav1.Add(expectedTotalUsed, sysUsed) + gqm.updateGroupDeltaUsedNoLock(extension.SystemQuotaName, sysUsed) + assert.Equal(t, sysUsed, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetUsed()) + + defaultUsed := createResourceList(2, 5) + expectedTotalUsed = quotav1.Add(expectedTotalUsed, defaultUsed) + gqm.updateGroupDeltaUsedNoLock(extension.DefaultQuotaName, defaultUsed) + assert.Equal(t, defaultUsed, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetUsed()) + + //case1: no quota, root quota used + rootQuotaUsed := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName).GetUsed() + sysAndDefaultUsed := quotav1.Add(sysUsed, defaultUsed) + assert.Equal(t, rootQuotaUsed, sysAndDefaultUsed) + + //case2: just pod no quota, root quota used + pod1 := schetesting.MakePod().Name("1").Obj() + pod1.Spec.Containers = []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: createResourceList(10, 10), + }, + }, + } + pod1Used := pod1.Spec.Containers[0].Resources.Requests + expectedTotalUsed = quotav1.Add(expectedTotalUsed, pod1Used) + gqm.updatePodCacheNoLock(extension.DefaultQuotaName, pod1, true) + assert.Equal(t, 1, len(gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetPodCache())) + gqm.updatePodIsAssignedNoLock(extension.DefaultQuotaName, pod1, true) + assert.True(t, gqm.getPodIsAssignedNoLock(extension.DefaultQuotaName, pod1)) + gqm.updatePodUsedNoLock(extension.DefaultQuotaName, nil, pod1) + assert.Equal(t, expectedTotalUsed, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed()) + + // case3: when build quota tree, root quota used + qi1 := createQuota("1", extension.RootQuotaName, 20, 20, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 30, 30, 8, 6) + qi3 := createQuota("3", "1", 30, 20, 8, 5) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + gqm.UpdateQuota(qi3, false) + gqm.updateGroupDeltaUsedNoLock("2", createResourceList(5, 5)) + gqm.updateGroupDeltaUsedNoLock("3", createResourceList(7, 5)) + + expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(5, 5)) + expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(7, 5)) + + rootQuotaUsed = gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed() + assert.Equal(t, expectedTotalUsed, rootQuotaUsed) +} + +func TestGroupQuotaManager_UpdateRootQuotaRequest(t *testing.T) { + expectedTotalRequest := createResourceList(0, 0) + gqm := NewGroupQuotaManagerForTest() + gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) + + sysRequest := createResourceList(10, 30) + expectedTotalRequest = quotav1.Add(expectedTotalRequest, sysRequest) + gqm.updateGroupDeltaRequestNoLock(extension.SystemQuotaName, sysRequest) + assert.Equal(t, sysRequest, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetRequest()) + + defaultRequest := createResourceList(2, 5) + expectedTotalRequest = quotav1.Add(expectedTotalRequest, defaultRequest) + gqm.updateGroupDeltaRequestNoLock(extension.DefaultQuotaName, defaultRequest) + assert.Equal(t, defaultRequest, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetRequest()) + + //case1: no other quota, root quota request + rootQuotaRequest := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName).GetRequest() + sysAndDefaultRequest := quotav1.Add(sysRequest, defaultRequest) + assert.Equal(t, rootQuotaRequest, sysAndDefaultRequest) + + //case2: just pod no quota, root quota request + pod1 := schetesting.MakePod().Name("1").Obj() + pod1.Spec.Containers = []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: createResourceList(10, 10), + }, + }, + } + pod1Request := pod1.Spec.Containers[0].Resources.Requests + expectedTotalRequest = quotav1.Add(expectedTotalRequest, pod1Request) + gqm.updatePodCacheNoLock(extension.DefaultQuotaName, pod1, true) + assert.Equal(t, 1, len(gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetPodCache())) + gqm.updatePodIsAssignedNoLock(extension.DefaultQuotaName, pod1, true) + assert.True(t, gqm.getPodIsAssignedNoLock(extension.DefaultQuotaName, pod1)) + gqm.updatePodRequestNoLock(extension.DefaultQuotaName, nil, pod1) + assert.Equal(t, expectedTotalRequest, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetRequest()) + + // case 3. when build quota tree, root quota request + qi1 := createQuota("1", extension.RootQuotaName, 20, 20, 10, 10) + qi2 := createQuota("2", extension.RootQuotaName, 30, 30, 8, 6) + qi3 := createQuota("3", "1", 30, 20, 8, 5) + gqm.UpdateQuota(qi1, false) + gqm.UpdateQuota(qi2, false) + gqm.UpdateQuota(qi3, false) + gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5)) + gqm.updateGroupDeltaRequestNoLock("3", createResourceList(7, 5)) + + expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(5, 5)) + expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(7, 5)) + + rootQuotaRequest = gqm.GetQuotaInfoByName(extension.RootQuotaName).GetRequest() + assert.Equal(t, expectedTotalRequest, rootQuotaRequest) +} + +func TestGroupQuotaManager_RootQuotaRuntime(t *testing.T) { + gqm := NewGroupQuotaManagerForTest() + gqm.UpdateClusterTotalResource(createResourceList(1000, 1000)) + + // case: test root quota runtime + assert.Equal(t, gqm.RefreshRuntime(extension.RootQuotaName), gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy()) +} diff --git a/pkg/scheduler/plugins/elasticquota/plugin.go b/pkg/scheduler/plugins/elasticquota/plugin.go index b2a55048bb..f0af1397df 100644 --- a/pkg/scheduler/plugins/elasticquota/plugin.go +++ b/pkg/scheduler/plugins/elasticquota/plugin.go @@ -116,6 +116,7 @@ func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error) ctx := context.TODO() + elasticQuota.createRootQuotaIfNotPresent() elasticQuota.createSystemQuotaIfNotPresent() elasticQuota.createDefaultQuotaIfNotPresent() frameworkexthelper.ForceSyncFromInformer(ctx.Done(), scheSharedInformerFactory, elasticQuotaInformer.Informer(), cache.ResourceEventHandlerFuncs{ diff --git a/pkg/scheduler/plugins/elasticquota/plugin_helper.go b/pkg/scheduler/plugins/elasticquota/plugin_helper.go index 7b1413fc12..64702bd6aa 100644 --- a/pkg/scheduler/plugins/elasticquota/plugin_helper.go +++ b/pkg/scheduler/plugins/elasticquota/plugin_helper.go @@ -144,6 +144,33 @@ func (g *Plugin) createSystemQuotaIfNotPresent() { klog.Infof("create SystemQuota successfully") } +// createRootQuotaIfNotPresent create RootQuotaGroup's CRD +func (g *Plugin) createRootQuotaIfNotPresent() { + eq, _ := g.quotaLister.ElasticQuotas(g.pluginArgs.QuotaGroupNamespace).Get(extension.RootQuotaName) + if eq != nil { + klog.Infof("RootQuota already exists, skip create it.") + return + } + rootElasticQuota := &schedulerv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: extension.RootQuotaName, + Namespace: g.pluginArgs.QuotaGroupNamespace, + Labels: make(map[string]string), + Annotations: make(map[string]string), + }, + } + rootElasticQuota.Labels[extension.LabelQuotaIsParent] = "true" + rootElasticQuota.Labels[extension.LabelAllowLentResource] = "false" + rootElasticQuota.Labels[extension.LabelQuotaParent] = "" + _, err := g.client.SchedulingV1alpha1().ElasticQuotas(g.pluginArgs.QuotaGroupNamespace). + Create(context.TODO(), rootElasticQuota, metav1.CreateOptions{}) + if err != nil { + klog.Errorf("create root group fail, err:%v", err.Error()) + return + } + klog.Infof("create RootQuota successfully") +} + func (g *Plugin) snapshotPostFilterState(quotaInfo *core.QuotaInfo, state *framework.CycleState) *PostFilterState { postFilterState := &PostFilterState{ quotaInfo: quotaInfo, diff --git a/pkg/scheduler/plugins/elasticquota/plugin_test.go b/pkg/scheduler/plugins/elasticquota/plugin_test.go index 79680f3c27..658e7f1ff1 100644 --- a/pkg/scheduler/plugins/elasticquota/plugin_test.go +++ b/pkg/scheduler/plugins/elasticquota/plugin_test.go @@ -1306,7 +1306,7 @@ func TestPlugin_Recover(t *testing.T) { assert.Equal(t, pl.groupQuotaManager.GetQuotaInfoByName("test1").GetRequest(), createResourceList(40, 40)) assert.Equal(t, pl.groupQuotaManager.GetQuotaInfoByName("test1").GetUsed(), createResourceList(40, 40)) assert.True(t, quotav1.IsZero(pl.groupQuotaManager.GetQuotaInfoByName(extension.DefaultQuotaName).GetRequest())) - assert.Equal(t, len(pl.groupQuotaManager.GetAllQuotaNames()), 4) + assert.Equal(t, len(pl.groupQuotaManager.GetAllQuotaNames()), 5) } func TestPlugin_migrateDefaultQuotaGroupsPod(t *testing.T) { diff --git a/pkg/webhook/elasticquota/quota_topology.go b/pkg/webhook/elasticquota/quota_topology.go index 43cf5f2491..382d317745 100644 --- a/pkg/webhook/elasticquota/quota_topology.go +++ b/pkg/webhook/elasticquota/quota_topology.go @@ -153,7 +153,7 @@ func (qt *quotaTopology) fillQuotaDefaultInformation(quota *v1alpha1.ElasticQuot quota.Annotations = make(map[string]string) } - if parentName, exist := quota.Labels[extension.LabelQuotaParent]; !exist || len(parentName) == 0 { + if parentName, exist := quota.Labels[extension.LabelQuotaParent]; (!exist || len(parentName) == 0) && quota.Name != extension.RootQuotaName { quota.Labels[extension.LabelQuotaParent] = extension.RootQuotaName klog.V(5).Infof("fill quota %v parent as root", quota.Name) } diff --git a/pkg/webhook/elasticquota/quota_topology_check.go b/pkg/webhook/elasticquota/quota_topology_check.go index ce33e1d8ce..991f847ce2 100644 --- a/pkg/webhook/elasticquota/quota_topology_check.go +++ b/pkg/webhook/elasticquota/quota_topology_check.go @@ -127,6 +127,9 @@ func (qt *quotaTopology) checkParentQuotaInfo(quotaName, parentName string) erro } func (qt *quotaTopology) checkSubAndParentGroupMaxQuotaKeySame(quotaInfo *QuotaInfo) error { + if quotaInfo.Name == extension.RootQuotaName { + return nil + } if quotaInfo.ParentName != extension.RootQuotaName { parentInfo := qt.quotaInfoMap[quotaInfo.ParentName] if !checkQuotaKeySame(parentInfo.CalculateInfo.Max, quotaInfo.CalculateInfo.Max) {