Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

koord-scheduler: update quota and pod handle for elasticquotaprofile #1621

Merged
merged 4 commits into from
Sep 11, 2023
Merged

koord-scheduler: update quota and pod handle for elasticquotaprofile #1621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b6bd6f
update quota and pod handler for multiple quota tree
Sep 6, 2023
016f900
refactor revoke controller for multi quota tree
Sep 8, 2023
d556f90
update webhook for quota tree id
Sep 8, 2023
4d8b28f
fix default group quota manager resource when root quota updated
Sep 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions apis/extension/elastic_quota.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,14 @@ const (
LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource"
LabelQuotaName = QuotaKoordinatorPrefix + "/name"
LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile"
LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root"
LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id"
AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight"
AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime"
AnnotationRequest = QuotaKoordinatorPrefix + "/request"
AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request"
AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys"
AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource"
AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
)

Expand All @@ -60,6 +63,10 @@ func IsAllowLentResource(quota *v1alpha1.ElasticQuota) bool {
return quota.Labels[LabelAllowLentResource] != "false"
}

func GetQuotaTreeID(quota *v1alpha1.ElasticQuota) string {
return quota.Labels[LabelQuotaTreeID]
}

func GetSharedWeight(quota *v1alpha1.ElasticQuota) corev1.ResourceList {
value, exist := quota.Annotations[AnnotationSharedWeight]
if exist {
Expand Down
44 changes: 43 additions & 1 deletion pkg/quota-controller/profile/profile_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package profile
import (
"context"
"encoding/json"
"fmt"
"hash/fnv"
"math"
"reflect"
"strconv"
Expand Down Expand Up @@ -81,6 +83,21 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

quotaTreeID := profile.Labels[extension.LabelQuotaTreeID]
if quotaTreeID == "" {
// generate quota tree id
quotaTreeID = hash(fmt.Sprintf("%s/%s", profile.Namespace, profile.Name))
if profile.Labels == nil {
profile.Labels = make(map[string]string)
}
profile.Labels[extension.LabelQuotaTreeID] = quotaTreeID
err := r.Client.Update(context.TODO(), profile)
if err != nil {
klog.Errorf("failed to update profile tree id: %v, error: %v", req.NamespacedName, err)
return ctrl.Result{Requeue: true}, err
}
}

selector, err := metav1.LabelSelectorAsSelector(profile.Spec.NodeSelector)
if err != nil {
klog.Errorf("failed to convert profile %v nodeSelector, error: %v", req.NamespacedName, err)
Expand Down Expand Up @@ -143,21 +160,40 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
value := MultiplyQuantity(quantity, resourceName, ratio)

min[resourceName] = value
totalResource[resourceName] = value
max[resourceName] = *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI)
}

// update min and max
quota.Spec.Min = min
quota.Spec.Max = max

if quota.Labels == nil {
quota.Labels = make(map[string]string)
}
// update quota label
quota.Labels[extension.LabelQuotaProfile] = profile.Name
if profile.Spec.QuotaLabels != nil {
for k, v := range profile.Spec.QuotaLabels {
quota.Labels[k] = v
}
}
// update quota tree id
quota.Labels[extension.LabelQuotaTreeID] = quotaTreeID

// update quota root label
quota.Labels[extension.LabelQuotaIsRoot] = "true"

// update total resource
data, err := json.Marshal(totalResource)
if err != nil {
klog.Errorf("failed marshal total resources, err: %v", err)
return ctrl.Result{Requeue: true}, err
}
if quota.Annotations == nil {
quota.Annotations = make(map[string]string)
}
quota.Annotations[extension.AnnotationTotalResource] = string(data)

if !quotaExist {
err = r.Client.Create(context.TODO(), quota)
Expand All @@ -167,7 +203,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{Requeue: true}, err
}
} else {
if !reflect.DeepEqual(quota.Labels, oldQuota.Labels) || !reflect.DeepEqual(quota.Spec, oldQuota.Spec) {
if !reflect.DeepEqual(quota.Labels, oldQuota.Labels) || !reflect.DeepEqual(quota.Annotations, oldQuota.Annotations) || !reflect.DeepEqual(quota.Spec, oldQuota.Spec) {
err = r.Client.Update(context.TODO(), quota)
if err != nil {
r.Recorder.Eventf(profile, "Warning", ReasonUpdateQuotaFailed, "failed to update quota, err: %s", err)
Expand Down Expand Up @@ -211,3 +247,9 @@ func MultiplyQuantity(value resource.Quantity, resName corev1.ResourceName, rati
}
return q
}

func hash(s string) string {
h := fnv.New64a()
h.Write([]byte(s))
return strconv.FormatUint(h.Sum64(), 10)
}
159 changes: 132 additions & 27 deletions pkg/quota-controller/profile/profile_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package profile

import (
"context"
"encoding/json"
"fmt"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -45,6 +47,16 @@ func createResourceList(cpu, mem int64) corev1.ResourceList {
}
}

func createResourceListWithStorage(cpu, mem, storage int64) corev1.ResourceList {
return corev1.ResourceList{
// use NewMilliQuantity to calculate the runtimeQuota correctly in cpu dimension
// when the request is smaller than 1 core.
corev1.ResourceCPU: *resource.NewMilliQuantity(cpu*1000, resource.DecimalSI),
corev1.ResourceMemory: *resource.NewQuantity(mem, resource.BinarySI),
corev1.ResourceStorage: *resource.NewQuantity(storage, resource.BinarySI),
}
}

func defaultCreateNode(nodeName string, labels map[string]string, capacity corev1.ResourceList) *corev1.Node {
return &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -64,19 +76,23 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
schedv1alpha1.AddToScheme(scheme)

nodes := []*corev1.Node{
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceList(10, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceList(10, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceList(10, 1000)),
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceListWithStorage(10, 1000, 1000)),
}

treeID1 := hash(fmt.Sprintf("%s/%s", "", "profile1"))
treeID2 := hash(fmt.Sprintf("%s/%s", "", "profile2"))

resourceRatio := "0.9"

tests := []struct {
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectQuotaLabels map[string]string
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectQuotaLabels map[string]string
}{
{
name: "cn-hangzhou-a profile",
Expand All @@ -91,9 +107,14 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile1"},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "cn-hangzhou-b profile",
Expand All @@ -108,18 +129,23 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile2"},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile2",
extension.LabelQuotaTreeID: treeID2,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "more quota labels",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile3",
Name: "profile1",
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile3-root",
QuotaName: "profile1-root",
QuotaLabels: map[string]string{
"topology.kubernetes.io/zone": "cn-hangzhou-a",
},
Expand All @@ -128,18 +154,24 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile3", "topology.kubernetes.io/zone": "cn-hangzhou-a"},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
"topology.kubernetes.io/zone": "cn-hangzhou-a",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "exist quota",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile4",
Name: "profile1",
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile4-root",
QuotaName: "profile1-root",
QuotaLabels: map[string]string{
"topology.kubernetes.io/zone": "cn-hangzhou-a",
},
Expand All @@ -150,15 +182,22 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
oriQuota: &schedv1alpha1.ElasticQuota{
ObjectMeta: metav1.ObjectMeta{
Name: "profile4-root",
Name: "profile1-root",
Labels: map[string]string{"a": "a"},
},
Spec: schedv1alpha1.ElasticQuotaSpec{
Min: createResourceList(5, 50),
},
},
expectQuotaMin: createResourceList(20, 2000),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile4", "topology.kubernetes.io/zone": "cn-hangzhou-a", "a": "a"},
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
"topology.kubernetes.io/zone": "cn-hangzhou-a",
"a": "a",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "has ratio",
Expand All @@ -174,9 +213,69 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectQuotaLabels: map[string]string{extension.LabelQuotaProfile: "profile1"},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "with tree id",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile1",
Labels: map[string]string{
extension.LabelQuotaTreeID: "tree1",
},
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile1-root",
ResourceRatio: &resourceRatio,
NodeSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"},
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
extension.LabelQuotaIsRoot: "true",
},
},
{
name: "with resource key",
profile: &quotav1alpha1.ElasticQuotaProfile{
ObjectMeta: metav1.ObjectMeta{
Name: "profile1",
Labels: map[string]string{
extension.LabelQuotaTreeID: "tree1",
},
Annotations: map[string]string{
extension.AnnotationResourceKeys: "[\"cpu\"]",
},
},
Spec: quotav1alpha1.ElasticQuotaProfileSpec{
QuotaName: "profile1-root",
ResourceRatio: &resourceRatio,
NodeSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"},
},
},
},
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 2000, 2000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
extension.LabelQuotaIsRoot: "true",
},
},
}

Expand Down Expand Up @@ -206,7 +305,13 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
quota := &schedv1alpha1.ElasticQuota{}
err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: tc.profile.Namespace, Name: tc.profile.Spec.QuotaName}, quota)
assert.NoError(t, err)

total := corev1.ResourceList{}
err = json.Unmarshal([]byte(quota.Annotations[extension.AnnotationTotalResource]), &total)
assert.NoError(t, err)

assert.True(t, quotav1.Equals(tc.expectQuotaMin, quota.Spec.Min))
assert.True(t, quotav1.Equals(tc.expectTotalResource, total))
assert.Equal(t, tc.expectQuotaLabels, quota.Labels)
})
}
Expand Down
Loading