Skip to content

Commit

Permalink
koord-scheduler: update quota and pod handle for elasticquotaprofile (#…
Browse files Browse the repository at this point in the history 
…1621)

Signed-off-by: chuanyun.lcy <[email protected]>
Co-authored-by: chuanyun.lcy <[email protected]>
  • Loading branch information
@shaloulcy
shaloulcy and chuanyun.lcy committed Sep 11, 2023
1 parent 4c1cfb3 commit 483e429
Show file tree
Hide file tree
Showing 25 changed files with 1,264 additions and 730 deletions.
7 changes: 7 additions & 0 deletions apis/extension/elastic_quota.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,14 @@ const (
LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource"
LabelQuotaName = QuotaKoordinatorPrefix + "/name"
LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile"
LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root"
LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id"
AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight"
AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime"
AnnotationRequest = QuotaKoordinatorPrefix + "/request"
AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request"
AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys"
AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource"
AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
)

Expand All @@ -60,6 +63,10 @@ func IsAllowLentResource(quota *v1alpha1.ElasticQuota) bool {
return quota.Labels[LabelAllowLentResource] != "false"
}

func GetQuotaTreeID(quota *v1alpha1.ElasticQuota) string {
return quota.Labels[LabelQuotaTreeID]
}

func GetSharedWeight(quota *v1alpha1.ElasticQuota) corev1.ResourceList {
value, exist := quota.Annotations[AnnotationSharedWeight]
if exist {
Expand Down
44 changes: 43 additions & 1 deletion pkg/quota-controller/profile/profile_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package profile
import (
"context"
"encoding/json"
"fmt"
"hash/fnv"
"math"
"reflect"
"strconv"
Expand Down Expand Up @@ -81,6 +83,21 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

quotaTreeID := profile.Labels[extension.LabelQuotaTreeID]
if quotaTreeID == "" {
// generate quota tree id
quotaTreeID = hash(fmt.Sprintf("%s/%s", profile.Namespace, profile.Name))
if profile.Labels == nil {
profile.Labels = make(map[string]string)
}
profile.Labels[extension.LabelQuotaTreeID] = quotaTreeID
err := r.Client.Update(context.TODO(), profile)
if err != nil {
klog.Errorf("failed to update profile tree id: %v, error: %v", req.NamespacedName, err)
return ctrl.Result{Requeue: true}, err
}
}

selector, err := metav1.LabelSelectorAsSelector(profile.Spec.NodeSelector)
if err != nil {
klog.Errorf("failed to convert profile %v nodeSelector, error: %v", req.NamespacedName, err)
Expand Down Expand Up @@ -143,21 +160,40 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
value := MultiplyQuantity(quantity, resourceName, ratio)

min[resourceName] = value
totalResource[resourceName] = value
max[resourceName] = *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI)
}

// update min and max
quota.Spec.Min = min
quota.Spec.Max = max

if quota.Labels == nil {
quota.Labels = make(map[string]string)
}
// update quota label
quota.Labels[extension.LabelQuotaProfile] = profile.Name
if profile.Spec.QuotaLabels != nil {
for k, v := range profile.Spec.QuotaLabels {
quota.Labels[k] = v
}
}
// update quota tree id
quota.Labels[extension.LabelQuotaTreeID] = quotaTreeID

// update quota root label
quota.Labels[extension.LabelQuotaIsRoot] = "true"

// update total resource
data, err := json.Marshal(totalResource)
if err != nil {
klog.Errorf("failed marshal total resources, err: %v", err)
return ctrl.Result{Requeue: true}, err
}
if quota.Annotations == nil {
quota.Annotations = make(map[string]string)
}
quota.Annotations[extension.AnnotationTotalResource] = string(data)

if !quotaExist {
err = r.Client.Create(context.TODO(), quota)
Expand All @@ -167,7 +203,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{Requeue: true}, err
}
} else {
if !reflect.DeepEqual(quota.Labels, oldQuota.Labels) || !reflect.DeepEqual(quota.Spec, oldQuota.Spec) {
if !reflect.DeepEqual(quota.Labels, oldQuota.Labels) || !reflect.DeepEqual(quota.Annotations, oldQuota.Annotations) || !reflect.DeepEqual(quota.Spec, oldQuota.Spec) {
err = r.Client.Update(context.TODO(), quota)
if err != nil {
r.Recorder.Eventf(profile, "Warning", ReasonUpdateQuotaFailed, "failed to update quota, err: %s", err)
Expand Down Expand Up @@ -211,3 +247,9 @@ func MultiplyQuantity(value resource.Quantity, resName corev1.ResourceName, rati
}
return q
}

func hash(s string) string {
h := fnv.New64a()
h.Write([]byte(s))
return strconv.FormatUint(h.Sum64(), 10)
}
159 changes: 132 additions & 27 deletions pkg/quota-controller/profile/profile_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package profile

import (
"context"
"encoding/json"
"fmt"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -45,6 +47,16 @@ func createResourceList(cpu, mem int64) corev1.ResourceList {
}
}

func createResourceListWithStorage(cpu, mem, storage int64) corev1.ResourceList {
return corev1.ResourceList{
// use NewMilliQuantity to calculate the runtimeQuota correctly in cpu dimension
// when the request is smaller than 1 core.
corev1.ResourceCPU: *resource.NewMilliQuantity(cpu*1000, resource.DecimalSI),
corev1.ResourceMemory: *resource.NewQuantity(mem, resource.BinarySI),
corev1.ResourceStorage: *resource.NewQuantity(storage, resource.BinarySI),
}
}

func defaultCreateNode(nodeName string, labels map[string]string, capacity corev1.ResourceList) *corev1.Node {
return &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -64,19 +76,23 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
schedv1alpha1.AddToScheme(scheme)

nodes := []*corev1.Node{
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceList(10, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceList(10, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceList(10, 1000)),
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceListWithStorage(10, 1000, 1000)),
}

treeID1 := hash(fmt.Sprintf("%s/%s", "", "profile1"))
treeID2 := hash(fmt.Sprintf("%s/%s", "", "profile2"))

resourceRatio := "0.9"

tests := []struct {
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectQuotaLabels map[string]string
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectQuotaLabels map[string]string
}{
{
name: "cn-hangzhou-a profile",
Expand All @@ -91,9 +107,14 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile1"},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "cn-hangzhou-b profile",
Expand All @@ -108,18 +129,23 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile2"},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile2",
extension.LabelQuotaTreeID: treeID2,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "more quota labels",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile3",
Name: "profile1",
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile3-root",
QuotaName: "profile1-root",
QuotaLabels: map[string]string{
"topology.kubernetes.io/zone": "cn-hangzhou-a",
},
Expand All @@ -128,18 +154,24 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile3", "topology.kubernetes.io/zone": "cn-hangzhou-a"},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
"topology.kubernetes.io/zone": "cn-hangzhou-a",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "exist quota",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile4",
Name: "profile1",
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile4-root",
QuotaName: "profile1-root",
QuotaLabels: map[string]string{
"topology.kubernetes.io/zone": "cn-hangzhou-a",
},
Expand All @@ -150,15 +182,22 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
oriQuota: &schedv1alpha1.ElasticQuota{
ObjectMeta: metav1.ObjectMeta{
Name: "profile4-root",
Name: "profile1-root",
Labels: map[string]string{"a": "a"},
},
Spec: schedv1alpha1.ElasticQuotaSpec{
Min: createResourceList(5, 50),
},
},
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile4", "topology.kubernetes.io/zone": "cn-hangzhou-a", "a": "a"},
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
"topology.kubernetes.io/zone": "cn-hangzhou-a",
"a": "a",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "has ratio",
Expand All @@ -174,9 +213,69 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile1"},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "with tree id",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile1",
Labels: map[string]string{
extension.LabelQuotaTreeID: "tree1",
},
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile1-root",
ResourceRatio: &resourceRatio,
NodeSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"},
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "with resource key",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile1",
Labels: map[string]string{
extension.LabelQuotaTreeID: "tree1",
},
Annotations: map[string]string{
extension.AnnotationResourceKeys: "[\"cpu\"]",
},
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile1-root",
ResourceRatio: &resourceRatio,
NodeSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"},
},
},
},
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
extension.LabelQuotaIsRoot: "true",
},
},
}

Expand Down Expand Up @@ -206,7 +305,13 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
quota := &schedv1alpha1.ElasticQuota{}
err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: tc.profile.Namespace, Name: tc.profile.Spec.QuotaName}, quota)
assert.NoError(t, err)

total := corev1.ResourceList{}
err = json.Unmarshal([]byte(quota.Annotations[extension.AnnotationTotalResource]), &total)
assert.NoError(t, err)

assert.True(t, quotav1.Equals(tc.expectQuotaMin, quota.Spec.Min))
assert.True(t, quotav1.Equals(tc.expectTotalResource, total))
assert.Equal(t, tc.expectQuotaLabels, quota.Labels)
})
}
Expand Down
Loading

0 comments on commit 483e429

Please sign in to comment.