koord-scheduler: export koordinator-root-quota as CRD object (#1555)

Signed-off-by: wangyang55 <[email protected]>

apis/extension/elastic_quota.go

@@ -42,7 +42,7 @@ const (
 
 func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string {
 	parentName := quota.Labels[LabelQuotaParent]
-	if parentName == "" {
+	if parentName == "" && quota.Name != RootQuotaName {
 		return RootQuotaName //default return RootQuotaName
 	}
 	return parentName

pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go

@@ -77,6 +77,7 @@ func NewGroupQuotaManager(systemGroupMax, defaultGroupMax v1.ResourceList, nodeL
 	quotaManager.quotaInfoMap[extension.SystemQuotaName].setMaxQuotaNoLock(systemGroupMax)
 	quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName)
 	quotaManager.quotaInfoMap[extension.DefaultQuotaName].setMaxQuotaNoLock(defaultGroupMax)
+	quotaManager.quotaInfoMap[extension.RootQuotaName] = NewQuotaInfo(true, false, extension.RootQuotaName, "")
 	quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName)
 	quotaManager.setScaleMinQuotaEnabled(true)
 	return quotaManager
@@ -148,7 +149,7 @@ func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq
 		curQuotaInfo := curToAllParInfos[i]
 		oldSubLimitReq := curQuotaInfo.getLimitRequestNoLock()
 		curQuotaInfo.addRequestNonNegativeNoLock(deltaReq)
-		if curQuotaInfo.Name == extension.SystemQuotaName || curQuotaInfo.Name == extension.DefaultQuotaName {
+		if curQuotaInfo.Name == extension.RootQuotaName {
 			return
 		}
 		newSubLimitReq := curQuotaInfo.getLimitRequestNoLock()
@@ -199,6 +200,10 @@ func (gqm *GroupQuotaManager) RefreshRuntimeNoLock(quotaName string) v1.Resource
 		return nil
 	}
 
+	if quotaName == extension.RootQuotaName {
+		return gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy()
+	}
+
 	if quotaName == extension.SystemQuotaName || quotaName == extension.DefaultQuotaName {
 		return quotaInfo.getMax()
 	}
@@ -210,6 +215,9 @@ func (gqm *GroupQuotaManager) RefreshRuntimeNoLock(quotaName string) v1.Resource
 	totalRes := gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy()
 	for i := len(curToAllParInfos) - 1; i >= 0; i-- {
 		quotaInfo = curToAllParInfos[i]
+		if quotaInfo.Name == extension.RootQuotaName {
+			continue
+		}
 		parRuntimeQuotaCalculator := gqm.getRuntimeQuotaCalculatorByNameNoLock(quotaInfo.ParentName)
 		if parRuntimeQuotaCalculator == nil {
 			klog.Errorf("treeWrapper not exist! parentQuotaName:%v", quotaInfo.ParentName)
@@ -265,7 +273,7 @@ func (gqm *GroupQuotaManager) getCurToAllParentGroupQuotaInfoNoLock(quotaName st
 
 	for true {
 		curToAllParInfos = append(curToAllParInfos, quotaInfo)
-		if quotaInfo.ParentName == extension.RootQuotaName {
+		if quotaInfo.Name == extension.RootQuotaName {
 			break
 		}
 
@@ -381,6 +389,7 @@ func (gqm *GroupQuotaManager) resetAllGroupQuotaNoLock() {
 	childRequestMap, childUsedMap := make(quotaResMapType), make(quotaResMapType)
 	for quotaName, topoNode := range gqm.quotaTopoNodeMap {
 		if quotaName == extension.RootQuotaName {
+			gqm.resetRootQuotaUsedAndRequest()
 			continue
 		}
 		topoNode.quotaInfo.lock.Lock()
@@ -791,3 +800,15 @@ func (gqm *GroupQuotaManager) ResyncNodes() {
 
 	// TODO: remove none-exist node
 }
+
+func (gqm *GroupQuotaManager) resetRootQuotaUsedAndRequest() {
+	rootQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName)
+	rootQuotaInfo.lock.Lock()
+	defer rootQuotaInfo.lock.Unlock()
+
+	systemQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.SystemQuotaName)
+	defaultQuotaInfo := gqm.getQuotaInfoByNameNoLock(extension.DefaultQuotaName)
+
+	rootQuotaInfo.CalculateInfo.Used = quotav1.Add(systemQuotaInfo.GetUsed(), defaultQuotaInfo.GetUsed())
+	rootQuotaInfo.CalculateInfo.Request = quotav1.Add(systemQuotaInfo.GetRequest(), defaultQuotaInfo.GetRequest())
+}

pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go

@@ -54,7 +54,7 @@ func TestGroupQuotaManager_QuotaAdd(t *testing.T) {
 	}
 
 	assert.Equal(t, 6, len(gqm.runtimeQuotaCalculatorMap))
-	assert.Equal(t, 7, len(gqm.quotaInfoMap))
+	assert.Equal(t, 8, len(gqm.quotaInfoMap))
 	assert.Equal(t, 2, len(gqm.resourceKeys))
 }
 
@@ -143,7 +143,7 @@ func TestGroupQuotaManager_UpdateQuota(t *testing.T) {
 	gqm := NewGroupQuotaManagerForTest()
 	quota := CreateQuota("test1", "test-parent", 64, 100*GigaByte, 50, 80*GigaByte, true, false)
 	gqm.UpdateQuota(quota, false)
-	assert.Equal(t, len(gqm.quotaInfoMap), 3)
+	assert.Equal(t, len(gqm.quotaInfoMap), 4)
 }
 
 func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) {
@@ -180,7 +180,7 @@ func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) {
 	quota2 := AddQuotaToManager(t, gqm, "test2", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, false)
 	quota3 := AddQuotaToManager(t, gqm, "test3", extension.RootQuotaName, 96, 160*GigaByte, 40, 40*GigaByte, true, false)
 	assert.Equal(t, 4, len(gqm.runtimeQuotaCalculatorMap))
-	assert.Equal(t, 5, len(gqm.quotaInfoMap))
+	assert.Equal(t, 6, len(gqm.quotaInfoMap))
 
 	err := gqm.UpdateQuota(quota1, true)
 	assert.Nil(t, err)
@@ -198,13 +198,13 @@ func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) {
 	assert.True(t, quotaInfo == nil)
 
 	assert.Equal(t, 1, len(gqm.runtimeQuotaCalculatorMap))
-	assert.Equal(t, 2, len(gqm.quotaInfoMap))
+	assert.Equal(t, 3, len(gqm.quotaInfoMap))
 
 	AddQuotaToManager(t, gqm, "youku", extension.RootQuotaName, 96, 160*GigaByte, 70, 70*GigaByte, true, false)
 	quotaInfo = gqm.GetQuotaInfoByName("youku")
 	assert.NotNil(t, quotaInfo)
 	assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap))
-	assert.Equal(t, 3, len(gqm.quotaInfoMap))
+	assert.Equal(t, 4, len(gqm.quotaInfoMap))
 }
 
 func TestGroupQuotaManager_UpdateQuotaDeltaRequest(t *testing.T) {
@@ -325,7 +325,7 @@ func TestGroupQuotaManager_MultiQuotaAdd(t *testing.T) {
 	AddQuotaToManager(t, gqm, "test2-sub2-1", "test2-sub2", 96, 160*GigaByte, 0, 0*GigaByte, true, false)
 
 	assert.Equal(t, 9, len(gqm.runtimeQuotaCalculatorMap))
-	assert.Equal(t, 10, len(gqm.quotaInfoMap))
+	assert.Equal(t, 11, len(gqm.quotaInfoMap))
 	assert.Equal(t, 2, len(gqm.resourceKeys))
 	assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName].quotaTree[v1.ResourceCPU].quotaNodes))
 	assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap["test2"].quotaTree[v1.ResourceCPU].quotaNodes))
@@ -1088,8 +1088,9 @@ func NewGroupQuotaManagerForTest() *GroupQuotaManager {
 		scaleMinQuotaManager:                    NewScaleMinQuotaManager(),
 		quotaTopoNodeMap:                        make(map[string]*QuotaTopoNode),
 	}
-	quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, "")
-	quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, "")
+	quotaManager.quotaInfoMap[extension.SystemQuotaName] = NewQuotaInfo(false, true, extension.SystemQuotaName, extension.RootQuotaName)
+	quotaManager.quotaInfoMap[extension.DefaultQuotaName] = NewQuotaInfo(false, true, extension.DefaultQuotaName, extension.RootQuotaName)
+	quotaManager.quotaInfoMap[extension.RootQuotaName] = NewQuotaInfo(true, false, extension.RootQuotaName, "")
 	quotaManager.runtimeQuotaCalculatorMap[extension.RootQuotaName] = NewRuntimeQuotaCalculator(extension.RootQuotaName)
 	return quotaManager
 }
@@ -1305,3 +1306,121 @@ func TestGroupQuotaManager_IsParent(t *testing.T) {
 	assert.False(t, qi2Info.IsParent)
 
 }
+
+func TestGroupQuotaManager_UpdateRootQuotaUsed(t *testing.T) {
+	expectedTotalUsed := createResourceList(0, 0)
+	gqm := NewGroupQuotaManagerForTest()
+	gqm.UpdateClusterTotalResource(createResourceList(1000, 1000))
+
+	sysUsed := createResourceList(10, 30)
+	expectedTotalUsed = quotav1.Add(expectedTotalUsed, sysUsed)
+	gqm.updateGroupDeltaUsedNoLock(extension.SystemQuotaName, sysUsed)
+	assert.Equal(t, sysUsed, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetUsed())
+
+	defaultUsed := createResourceList(2, 5)
+	expectedTotalUsed = quotav1.Add(expectedTotalUsed, defaultUsed)
+	gqm.updateGroupDeltaUsedNoLock(extension.DefaultQuotaName, defaultUsed)
+	assert.Equal(t, defaultUsed, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetUsed())
+
+	//case1: no quota, root quota used
+	rootQuotaUsed := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName).GetUsed()
+	sysAndDefaultUsed := quotav1.Add(sysUsed, defaultUsed)
+	assert.Equal(t, rootQuotaUsed, sysAndDefaultUsed)
+
+	//case2: just pod no quota, root quota used
+	pod1 := schetesting.MakePod().Name("1").Obj()
+	pod1.Spec.Containers = []v1.Container{
+		{
+			Resources: v1.ResourceRequirements{
+				Requests: createResourceList(10, 10),
+			},
+		},
+	}
+	pod1Used := pod1.Spec.Containers[0].Resources.Requests
+	expectedTotalUsed = quotav1.Add(expectedTotalUsed, pod1Used)
+	gqm.updatePodCacheNoLock(extension.DefaultQuotaName, pod1, true)
+	assert.Equal(t, 1, len(gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetPodCache()))
+	gqm.updatePodIsAssignedNoLock(extension.DefaultQuotaName, pod1, true)
+	assert.True(t, gqm.getPodIsAssignedNoLock(extension.DefaultQuotaName, pod1))
+	gqm.updatePodUsedNoLock(extension.DefaultQuotaName, nil, pod1)
+	assert.Equal(t, expectedTotalUsed, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed())
+
+	// case3: when build quota tree, root quota used
+	qi1 := createQuota("1", extension.RootQuotaName, 20, 20, 10, 10)
+	qi2 := createQuota("2", extension.RootQuotaName, 30, 30, 8, 6)
+	qi3 := createQuota("3", "1", 30, 20, 8, 5)
+	gqm.UpdateQuota(qi1, false)
+	gqm.UpdateQuota(qi2, false)
+	gqm.UpdateQuota(qi3, false)
+	gqm.updateGroupDeltaUsedNoLock("2", createResourceList(5, 5))
+	gqm.updateGroupDeltaUsedNoLock("3", createResourceList(7, 5))
+
+	expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(5, 5))
+	expectedTotalUsed = quotav1.Add(expectedTotalUsed, createResourceList(7, 5))
+
+	rootQuotaUsed = gqm.GetQuotaInfoByName(extension.RootQuotaName).GetUsed()
+	assert.Equal(t, expectedTotalUsed, rootQuotaUsed)
+}
+
+func TestGroupQuotaManager_UpdateRootQuotaRequest(t *testing.T) {
+	expectedTotalRequest := createResourceList(0, 0)
+	gqm := NewGroupQuotaManagerForTest()
+	gqm.UpdateClusterTotalResource(createResourceList(1000, 1000))
+
+	sysRequest := createResourceList(10, 30)
+	expectedTotalRequest = quotav1.Add(expectedTotalRequest, sysRequest)
+	gqm.updateGroupDeltaRequestNoLock(extension.SystemQuotaName, sysRequest)
+	assert.Equal(t, sysRequest, gqm.GetQuotaInfoByName(extension.SystemQuotaName).GetRequest())
+
+	defaultRequest := createResourceList(2, 5)
+	expectedTotalRequest = quotav1.Add(expectedTotalRequest, defaultRequest)
+	gqm.updateGroupDeltaRequestNoLock(extension.DefaultQuotaName, defaultRequest)
+	assert.Equal(t, defaultRequest, gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetRequest())
+
+	//case1: no other quota, root quota request
+	rootQuotaRequest := gqm.getQuotaInfoByNameNoLock(extension.RootQuotaName).GetRequest()
+	sysAndDefaultRequest := quotav1.Add(sysRequest, defaultRequest)
+	assert.Equal(t, rootQuotaRequest, sysAndDefaultRequest)
+
+	//case2: just pod no quota, root quota request
+	pod1 := schetesting.MakePod().Name("1").Obj()
+	pod1.Spec.Containers = []v1.Container{
+		{
+			Resources: v1.ResourceRequirements{
+				Requests: createResourceList(10, 10),
+			},
+		},
+	}
+	pod1Request := pod1.Spec.Containers[0].Resources.Requests
+	expectedTotalRequest = quotav1.Add(expectedTotalRequest, pod1Request)
+	gqm.updatePodCacheNoLock(extension.DefaultQuotaName, pod1, true)
+	assert.Equal(t, 1, len(gqm.GetQuotaInfoByName(extension.DefaultQuotaName).GetPodCache()))
+	gqm.updatePodIsAssignedNoLock(extension.DefaultQuotaName, pod1, true)
+	assert.True(t, gqm.getPodIsAssignedNoLock(extension.DefaultQuotaName, pod1))
+	gqm.updatePodRequestNoLock(extension.DefaultQuotaName, nil, pod1)
+	assert.Equal(t, expectedTotalRequest, gqm.GetQuotaInfoByName(extension.RootQuotaName).GetRequest())
+
+	// case 3. when build quota tree, root quota request
+	qi1 := createQuota("1", extension.RootQuotaName, 20, 20, 10, 10)
+	qi2 := createQuota("2", extension.RootQuotaName, 30, 30, 8, 6)
+	qi3 := createQuota("3", "1", 30, 20, 8, 5)
+	gqm.UpdateQuota(qi1, false)
+	gqm.UpdateQuota(qi2, false)
+	gqm.UpdateQuota(qi3, false)
+	gqm.updateGroupDeltaRequestNoLock("2", createResourceList(5, 5))
+	gqm.updateGroupDeltaRequestNoLock("3", createResourceList(7, 5))
+
+	expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(5, 5))
+	expectedTotalRequest = quotav1.Add(expectedTotalRequest, createResourceList(7, 5))
+
+	rootQuotaRequest = gqm.GetQuotaInfoByName(extension.RootQuotaName).GetRequest()
+	assert.Equal(t, expectedTotalRequest, rootQuotaRequest)
+}
+
+func TestGroupQuotaManager_RootQuotaRuntime(t *testing.T) {
+	gqm := NewGroupQuotaManagerForTest()
+	gqm.UpdateClusterTotalResource(createResourceList(1000, 1000))
+
+	// case: test root quota runtime
+	assert.Equal(t, gqm.RefreshRuntime(extension.RootQuotaName), gqm.totalResourceExceptSystemAndDefaultUsed.DeepCopy())
+}

pkg/scheduler/plugins/elasticquota/plugin.go

@@ -116,6 +116,7 @@ func New(args runtime.Object, handle framework.Handle) (framework.Plugin, error)
 
 	ctx := context.TODO()
 
+	elasticQuota.createRootQuotaIfNotPresent()
 	elasticQuota.createSystemQuotaIfNotPresent()
 	elasticQuota.createDefaultQuotaIfNotPresent()
 	frameworkexthelper.ForceSyncFromInformer(ctx.Done(), scheSharedInformerFactory, elasticQuotaInformer.Informer(), cache.ResourceEventHandlerFuncs{

pkg/scheduler/plugins/elasticquota/plugin_helper.go

@@ -144,6 +144,33 @@ func (g *Plugin) createSystemQuotaIfNotPresent() {
 	klog.Infof("create SystemQuota successfully")
 }
 
+// createRootQuotaIfNotPresent create RootQuotaGroup's CRD
+func (g *Plugin) createRootQuotaIfNotPresent() {
+	eq, _ := g.quotaLister.ElasticQuotas(g.pluginArgs.QuotaGroupNamespace).Get(extension.RootQuotaName)
+	if eq != nil {
+		klog.Infof("RootQuota already exists, skip create it.")
+		return
+	}
+	rootElasticQuota := &schedulerv1alpha1.ElasticQuota{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        extension.RootQuotaName,
+			Namespace:   g.pluginArgs.QuotaGroupNamespace,
+			Labels:      make(map[string]string),
+			Annotations: make(map[string]string),
+		},
+	}
+	rootElasticQuota.Labels[extension.LabelQuotaIsParent] = "true"
+	rootElasticQuota.Labels[extension.LabelAllowLentResource] = "false"
+	rootElasticQuota.Labels[extension.LabelQuotaParent] = ""
+	_, err := g.client.SchedulingV1alpha1().ElasticQuotas(g.pluginArgs.QuotaGroupNamespace).
+		Create(context.TODO(), rootElasticQuota, metav1.CreateOptions{})
+	if err != nil {
+		klog.Errorf("create root group fail, err:%v", err.Error())
+		return
+	}
+	klog.Infof("create RootQuota successfully")
+}
+
 func (g *Plugin) snapshotPostFilterState(quotaInfo *core.QuotaInfo, state *framework.CycleState) *PostFilterState {
 	postFilterState := &PostFilterState{
 		quotaInfo: quotaInfo,

pkg/scheduler/plugins/elasticquota/plugin_test.go

@@ -1306,7 +1306,7 @@ func TestPlugin_Recover(t *testing.T) {
 	assert.Equal(t, pl.groupQuotaManager.GetQuotaInfoByName("test1").GetRequest(), createResourceList(40, 40))
 	assert.Equal(t, pl.groupQuotaManager.GetQuotaInfoByName("test1").GetUsed(), createResourceList(40, 40))
 	assert.True(t, quotav1.IsZero(pl.groupQuotaManager.GetQuotaInfoByName(extension.DefaultQuotaName).GetRequest()))
-	assert.Equal(t, len(pl.groupQuotaManager.GetAllQuotaNames()), 4)
+	assert.Equal(t, len(pl.groupQuotaManager.GetAllQuotaNames()), 5)
 }
 
 func TestPlugin_migrateDefaultQuotaGroupsPod(t *testing.T) {

pkg/webhook/elasticquota/quota_topology.go

@@ -153,7 +153,7 @@ func (qt *quotaTopology) fillQuotaDefaultInformation(quota *v1alpha1.ElasticQuot
 		quota.Annotations = make(map[string]string)
 	}
 
-	if parentName, exist := quota.Labels[extension.LabelQuotaParent]; !exist || len(parentName) == 0 {
+	if parentName, exist := quota.Labels[extension.LabelQuotaParent]; (!exist || len(parentName) == 0) && quota.Name != extension.RootQuotaName {
 		quota.Labels[extension.LabelQuotaParent] = extension.RootQuotaName
 		klog.V(5).Infof("fill quota %v parent as root", quota.Name)
 	}

pkg/webhook/elasticquota/quota_topology_check.go

@@ -127,6 +127,9 @@ func (qt *quotaTopology) checkParentQuotaInfo(quotaName, parentName string) erro
 }
 
 func (qt *quotaTopology) checkSubAndParentGroupMaxQuotaKeySame(quotaInfo *QuotaInfo) error {
+	if quotaInfo.Name == extension.RootQuotaName {
+		return nil
+	}
 	if quotaInfo.ParentName != extension.RootQuotaName {
 		parentInfo := qt.quotaInfoMap[quotaInfo.ParentName]
 		if !checkQuotaKeySame(parentInfo.CalculateInfo.Max, quotaInfo.CalculateInfo.Max) {