From f681940ee05276e944f7dfef315af486a681f9c5 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 2 Feb 2024 16:36:32 +0100 Subject: [PATCH 01/36] feat(in-cluster): update example to use native crossplane way Signed-off-by: Christopher Haar Signed-off-by: Alexander Brandstedt --- examples/provider/config-in-cluster.yaml | 31 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/examples/provider/config-in-cluster.yaml b/examples/provider/config-in-cluster.yaml index 3150c45f..083a6384 100644 --- a/examples/provider/config-in-cluster.yaml +++ b/examples/provider/config-in-cluster.yaml @@ -1,8 +1,3 @@ -# Make sure provider-kubernetes has enough permissions to install your objects into cluster -# -# You can give admin permissions by running: -# SA=$(kubectl -n crossplane-system get sa -o name | grep provider-kubernetes | sed -e 's|serviceaccount\/|crossplane-system:|g') -# kubectl create clusterrolebinding provider-kubernetes-admin-binding --clusterrole cluster-admin --serviceaccount="${SA}" apiVersion: kubernetes.crossplane.io/v1alpha1 kind: ProviderConfig metadata: @@ -10,3 +5,29 @@ metadata: spec: credentials: source: InjectedIdentity + runtimeConfigRef: + apiVersion: pkg.crossplane.io/v1beta1 + kind: DeploymentRuntimeConfig + name: provider-kubernetes +--- +apiVersion: pkg.crossplane.io/v1beta1 +kind: DeploymentRuntimeConfig +metadata: + name: provider-kubernetes +spec: + serviceAccountTemplate: + metadata: + name: provider-kubernetes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: provider-kubernetes-cluster-admin +subjects: + - kind: ServiceAccount + name: provider-kubernetes + namespace: crossplane-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io From 2cb7d756dbb1d6d09de68de6da300802587e8bf0 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 2 Feb 2024 16:55:58 +0100 Subject: [PATCH 02/36] feat(in-cluster): update example to use native crossplane way Signed-off-by: Christopher Haar Signed-off-by: Alexander Brandstedt --- examples/provider/config-in-cluster.yaml | 29 ++----------------- examples/provider/provider-in-cluster.yaml | 33 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 examples/provider/provider-in-cluster.yaml diff --git a/examples/provider/config-in-cluster.yaml b/examples/provider/config-in-cluster.yaml index 083a6384..3d7591fc 100644 --- a/examples/provider/config-in-cluster.yaml +++ b/examples/provider/config-in-cluster.yaml @@ -1,33 +1,8 @@ +## check provider-in-cluster.yaml for grant access apiVersion: kubernetes.crossplane.io/v1alpha1 kind: ProviderConfig metadata: name: kubernetes-provider spec: credentials: - source: InjectedIdentity - runtimeConfigRef: - apiVersion: pkg.crossplane.io/v1beta1 - kind: DeploymentRuntimeConfig - name: provider-kubernetes ---- -apiVersion: pkg.crossplane.io/v1beta1 -kind: DeploymentRuntimeConfig -metadata: - name: provider-kubernetes -spec: - serviceAccountTemplate: - metadata: - name: provider-kubernetes ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: provider-kubernetes-cluster-admin -subjects: - - kind: ServiceAccount - name: provider-kubernetes - namespace: crossplane-system -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io + source: InjectedIdentity \ No newline at end of file diff --git a/examples/provider/provider-in-cluster.yaml b/examples/provider/provider-in-cluster.yaml new file mode 100644 index 00000000..52988f19 --- /dev/null +++ b/examples/provider/provider-in-cluster.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: pkg.crossplane.io/v1 +kind: Provider +metadata: + name: provider-kubernetes +spec: + package: xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.10.0 + runtimeConfigRef: + apiVersion: pkg.crossplane.io/v1beta1 + kind: DeploymentRuntimeConfig + name: provider-kubernetes +--- +apiVersion: pkg.crossplane.io/v1beta1 +kind: DeploymentRuntimeConfig +metadata: + name: provider-kubernetes +spec: + serviceAccountTemplate: + metadata: + name: provider-kubernetes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: provider-kubernetes-cluster-admin +subjects: + - kind: ServiceAccount + name: provider-kubernetes + namespace: crossplane-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io From 7e79c253b99acdb1ca85bbe09a93a3a8a96875d1 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 8 Feb 2024 15:57:12 +0300 Subject: [PATCH 03/36] Update version in incluster example Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- examples/provider/config-in-cluster.yaml | 2 +- examples/provider/provider-in-cluster.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/provider/config-in-cluster.yaml b/examples/provider/config-in-cluster.yaml index 3d7591fc..0133ac72 100644 --- a/examples/provider/config-in-cluster.yaml +++ b/examples/provider/config-in-cluster.yaml @@ -1,4 +1,4 @@ -## check provider-in-cluster.yaml for grant access +# Check ./provider-in-cluster.yaml to see how to grant permissions to the Provider apiVersion: kubernetes.crossplane.io/v1alpha1 kind: ProviderConfig metadata: diff --git a/examples/provider/provider-in-cluster.yaml b/examples/provider/provider-in-cluster.yaml index 52988f19..b5e6b4be 100644 --- a/examples/provider/provider-in-cluster.yaml +++ b/examples/provider/provider-in-cluster.yaml @@ -4,7 +4,7 @@ kind: Provider metadata: name: provider-kubernetes spec: - package: xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.10.0 + package: xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.11.4 runtimeConfigRef: apiVersion: pkg.crossplane.io/v1beta1 kind: DeploymentRuntimeConfig From db6d9513ab2794d1abc1a6e35fca849ad741041e Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 8 Feb 2024 16:08:20 +0300 Subject: [PATCH 04/36] Bump dependencies Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- apis/object/v1alpha1/conversion_test.go | 18 +- go.mod | 67 ++- go.sum | 564 +++--------------- internal/controller/object/object_test.go | 4 +- .../kubernetes.crossplane.io_objects.yaml | 514 +++++++++------- ...ernetes.crossplane.io_providerconfigs.yaml | 76 ++- ...es.crossplane.io_providerconfigusages.yaml | 37 +- 7 files changed, 487 insertions(+), 793 deletions(-) diff --git a/apis/object/v1alpha1/conversion_test.go b/apis/object/v1alpha1/conversion_test.go index 3d46176f..122a93fe 100644 --- a/apis/object/v1alpha1/conversion_test.go +++ b/apis/object/v1alpha1/conversion_test.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" v1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/test" @@ -86,9 +86,9 @@ func TestConvertTo(t *testing.T) { Name: "topsecret", Namespace: "coolns", }, - FieldPath: pointer.String("data.password"), + FieldPath: ptr.To("data.password"), }, - ToFieldPath: pointer.String("data"), + ToFieldPath: ptr.To("data"), }, }, Readiness: v1alpha1.Readiness{Policy: v1alpha1.ReadinessPolicySuccessfulCreate}, @@ -132,9 +132,9 @@ func TestConvertTo(t *testing.T) { Name: "topsecret", Namespace: "coolns", }, - FieldPath: pointer.String("data.password"), + FieldPath: ptr.To("data.password"), }, - ToFieldPath: pointer.String("data"), + ToFieldPath: ptr.To("data"), }, }, Readiness: v1alpha2.Readiness{Policy: v1alpha2.ReadinessPolicySuccessfulCreate}, @@ -333,9 +333,9 @@ func TestConvertFrom(t *testing.T) { Name: "topsecret", Namespace: "coolns", }, - FieldPath: pointer.String("data.password"), + FieldPath: ptr.To("data.password"), }, - ToFieldPath: pointer.String("data"), + ToFieldPath: ptr.To("data"), }, }, Readiness: v1alpha2.Readiness{Policy: v1alpha2.ReadinessPolicySuccessfulCreate}, @@ -379,9 +379,9 @@ func TestConvertFrom(t *testing.T) { Name: "topsecret", Namespace: "coolns", }, - FieldPath: pointer.String("data.password"), + FieldPath: ptr.To("data.password"), }, - ToFieldPath: pointer.String("data"), + ToFieldPath: ptr.To("data"), }, }, Readiness: v1alpha1.Readiness{Policy: v1alpha1.ReadinessPolicySuccessfulCreate}, diff --git a/go.mod b/go.mod index d014c879..4ddb2dbb 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/crossplane-contrib/provider-kubernetes -go 1.20 +go 1.21 require ( github.com/Azure/kubelogin v0.0.0-00010101000000-000000000000 - github.com/crossplane/crossplane-runtime v1.14.4 + github.com/crossplane/crossplane-runtime v1.15.0-rc.1 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 github.com/google/go-cmp v0.6.0 github.com/pkg/errors v0.9.1 @@ -12,16 +12,16 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/oauth2 v0.16.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 - k8s.io/api v0.28.4 - k8s.io/apimachinery v0.28.4 - k8s.io/client-go v0.28.4 - k8s.io/utils v0.0.0-20230505201702-9f6742963106 - sigs.k8s.io/controller-runtime v0.16.3 - sigs.k8s.io/controller-tools v0.13.0 + k8s.io/api v0.29.1 + k8s.io/apimachinery v0.29.1 + k8s.io/client-go v0.29.1 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + sigs.k8s.io/controller-runtime v0.17.0 + sigs.k8s.io/controller-tools v0.14.0 ) require ( - cloud.google.com/go/compute v1.20.1 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect @@ -41,11 +41,12 @@ require ( github.com/dave/jennifer v1.4.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -65,42 +66,42 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.13.0 // indirect + golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/retry.v1 v1.0.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.3 // indirect - k8s.io/component-base v0.28.3 // indirect + k8s.io/apiextensions-apiserver v0.29.1 // indirect + k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) // This is a workaround until kubelogin project supports being consumed as a go module diff --git a/go.sum b/go.sum index cc2711c3..635bce6e 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,9 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= -cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= @@ -66,29 +28,18 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crossplane/crossplane-runtime v1.14.4 h1:64zSZ75g1QXIMxR2zSQvz4+TTSq5qCUU5lmpiVovVKE= -github.com/crossplane/crossplane-runtime v1.14.4/go.mod h1:aOP+5W2wKpvthVs3pFNbVOe1jwrKYbJho0ThGNCVz9o= +github.com/crossplane/crossplane-runtime v1.15.0-rc.1 h1:LkYZk/mRLZiJrL9gx6mM8t0Mvloy0nkfwy/BKmpC2wA= +github.com/crossplane/crossplane-runtime v1.15.0-rc.1/go.mod h1:kRcJjJQmBFrR2n/KhwL8wYS7xNfq3D8eK4JliEScOHI= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 h1:HigXs5tEQxWz0fcj8hzbU2UAZgEM7wPe0XRFOsrtF8Y= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79/go.mod h1:+e4OaFlOcmr0JvINHl/yvEYBrZawzTgj6pQumOH1SS0= github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= @@ -97,31 +48,24 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.2.2 h1:xfmOhhoH5fGPgbEAlhLpJH9p0z/0Qizio9osmvn9IUY= github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -129,6 +73,7 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -138,99 +83,42 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -242,10 +130,10 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -254,35 +142,36 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM= github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -290,190 +179,68 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/upbound/kubelogin v0.0.34-hotfix.1 h1:6Rmf1kVhBryriFc81O88rRQJ5oJ3HwsZeBKTnPi1oiY= github.com/upbound/kubelogin v0.0.34-hotfix.1/go.mod h1:lblMxK5B8o+CbJWdeoAb0K2r4rziMcW1b53qn6TTc38= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= +golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -481,170 +248,31 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= @@ -652,16 +280,16 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/dnaeon/go-vcr.v3 v3.1.2 h1:F1smfXBqQqwpVifDfUBQG6zzaGjzT+EnVZakrOdr5wA= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/dnaeon/go-vcr.v3 v3.1.2/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs= gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -669,39 +297,29 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= -k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= -k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= -k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= -k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= +k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= +k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= +k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= +k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= +k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= +k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= +k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= +k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= -k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= -sigs.k8s.io/controller-tools v0.13.0 h1:NfrvuZ4bxyolhDBt/rCZhDnx3M2hzlhgo5n3Iv2RykI= -sigs.k8s.io/controller-tools v0.13.0/go.mod h1:5vw3En2NazbejQGCeWKRrE7q4P+CW8/klfVqP8QZkgA= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-tools v0.14.0 h1:rnNoCC5wSXlrNoBKKzL70LNJKIQKEzT6lloG6/LF73A= +sigs.k8s.io/controller-tools v0.14.0/go.mod h1:TV7uOtNNnnR72SpzhStvPkoS/U5ir0nMudrkrC4M9Sc= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 2373366a..9b1e76e2 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -33,7 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" @@ -819,7 +819,7 @@ func Test_helmExternal_Observe(t *testing.T) { args: args{ mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = objectReferences() - obj.Spec.References[0].PatchesFrom.FieldPath = pointer.String("nonexistent_field") + obj.Spec.References[0].PatchesFrom.FieldPath = ptr.To("nonexistent_field") }), client: resource.ClientApplicator{ Client: &test.MockClient{ diff --git a/package/crds/kubernetes.crossplane.io_objects.yaml b/package/crds/kubernetes.crossplane.io_objects.yaml index 9e8f2a5f..4a652679 100644 --- a/package/crds/kubernetes.crossplane.io_objects.yaml +++ b/package/crds/kubernetes.crossplane.io_objects.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: objects.kubernetes.crossplane.io spec: conversion: @@ -57,18 +57,24 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: 'A Object is an provider Kubernetes API type Deprecated: v1alpha1.Object - is deprecated in favor of v1alpha2.Object' + description: |- + A Object is an provider Kubernetes API type + Deprecated: v1alpha1.Object is deprecated in favor of v1alpha2.Object properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -84,48 +90,56 @@ spec: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string toConnectionSecretKey: type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic type: array deletionPolicy: default: Delete - description: 'DeletionPolicy specifies what will happen to the underlying - external when this managed resource is deleted - either "Delete" - or "Orphan" the external resource. This field is planned to be deprecated - in favor of the ManagementPolicy field in a future release. Currently, - both could be set independently and non-default values would be - honored if the feature flag is enabled. See the design doc for more - information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223' + description: |- + DeletionPolicy specifies what will happen to the underlying external + when this managed resource is deleted - either "Delete" or "Orphan" the + external resource. + This field is planned to be deprecated in favor of the ManagementPolicy + field in a future release. Currently, both could be set independently and + non-default values would be honored if the feature flag is enabled. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 enum: - Orphan - Delete @@ -144,9 +158,9 @@ spec: type: object managementPolicy: default: Default - description: A ManagementPolicy determines what should happen to the - underlying external resource when a managed resource is created, - updated, deleted, or observed. + description: |- + A ManagementPolicy determines what should happen to the underlying external + resource when a managed resource is created, updated, deleted, or observed. enum: - Default - ObserveCreateUpdate @@ -156,9 +170,10 @@ spec: providerConfigRef: default: name: default - description: ProviderConfigReference specifies how the provider that - will be used to create, observe, update, and delete this managed - resource should be configured. + description: |- + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. properties: name: description: Name of the referenced object. @@ -168,21 +183,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of this - reference is required. The default is 'Required', which - means the reconcile will fail if the reference cannot be - resolved. 'Optional' means this reference will be a no-op - if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should - be resolved. The default is 'IfNotPresent', which will attempt - to resolve the reference only when the corresponding field - is not present. Use 'Always' to resolve the reference on - every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent @@ -192,9 +207,10 @@ spec: - name type: object providerRef: - description: 'ProviderReference specifies the provider that will be - used to create, observe, update, and delete this managed resource. - Deprecated: Please use ProviderConfigReference, i.e. `providerConfigRef`' + description: |- + ProviderReference specifies the provider that will be used to create, + observe, update, and delete this managed resource. + Deprecated: Please use ProviderConfigReference, i.e. `providerConfigRef` properties: name: description: Name of the referenced object. @@ -204,21 +220,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of this - reference is required. The default is 'Required', which - means the reconcile will fail if the reference cannot be - resolved. 'Optional' means this reference will be a no-op - if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should - be resolved. The default is 'IfNotPresent', which will attempt - to resolve the reference only when the corresponding field - is not present. Use 'Always' to resolve the reference on - every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent @@ -228,17 +244,19 @@ spec: - name type: object publishConnectionDetailsTo: - description: PublishConnectionDetailsTo specifies the connection secret - config which contains a name, metadata and a reference to secret - store config to which any connection details for this managed resource - should be written. Connection details frequently include the endpoint, - username, and password required to connect to the managed resource. + description: |- + PublishConnectionDetailsTo specifies the connection secret config which + contains a name, metadata and a reference to secret store config to + which any connection details for this managed resource should be written. + Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. properties: configRef: default: name: default - description: SecretStoreConfigRef specifies which secret store - config should be used for this ConnectionSecret. + description: |- + SecretStoreConfigRef specifies which secret store config should be used + for this ConnectionSecret. properties: name: description: Name of the referenced object. @@ -248,21 +266,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of - this reference is required. The default is 'Required', - which means the reconcile will fail if the reference - cannot be resolved. 'Optional' means this reference - will be a no-op if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should - be resolved. The default is 'IfNotPresent', which will - attempt to resolve the reference only when the corresponding - field is not present. Use 'Always' to resolve the reference - on every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent @@ -277,21 +295,22 @@ spec: annotations: additionalProperties: type: string - description: Annotations are the annotations to be added to - connection secret. - For Kubernetes secrets, this will be - used as "metadata.annotations". - It is up to Secret Store - implementation for others store types. + description: |- + Annotations are the annotations to be added to connection secret. + - For Kubernetes secrets, this will be used as "metadata.annotations". + - It is up to Secret Store implementation for others store types. type: object labels: additionalProperties: type: string - description: Labels are the labels/tags to be added to connection - secret. - For Kubernetes secrets, this will be used as "metadata.labels". - - It is up to Secret Store implementation for others store - types. + description: |- + Labels are the labels/tags to be added to connection secret. + - For Kubernetes secrets, this will be used as "metadata.labels". + - It is up to Secret Store implementation for others store types. type: object type: - description: Type is the SecretType for the connection secret. + description: |- + Type is the SecretType for the connection secret. - Only valid for Kubernetes Secret Stores. type: string type: object @@ -302,9 +321,10 @@ spec: - name type: object readiness: - description: Readiness defines how the object's readiness condition - should be computed, if not specified it will be considered ready - as soon as the underlying external resource is considered up-to-date. + description: |- + Readiness defines how the object's readiness condition should be computed, + if not specified it will be considered ready as soon as the underlying external + resource is considered up-to-date. properties: policy: default: SuccessfulCreate @@ -318,13 +338,14 @@ spec: type: object references: items: - description: Reference refers to an Object or arbitrary Kubernetes - resource and optionally patch values from that resource to the - current Object. + description: |- + Reference refers to an Object or arbitrary Kubernetes resource and optionally + patch values from that resource to the current Object. properties: dependsOn: - description: DependsOn is used to declare dependency on other - Object or arbitrary Kubernetes resource. + description: |- + DependsOn is used to declare dependency on other Object or arbitrary + Kubernetes resource. properties: apiVersion: default: kubernetes.crossplane.io/v1alpha1 @@ -344,17 +365,18 @@ spec: - name type: object patchesFrom: - description: PatchesFrom is used to declare dependency on other - Object or arbitrary Kubernetes resource, and also patch fields - from this object. + description: |- + PatchesFrom is used to declare dependency on other Object or arbitrary + Kubernetes resource, and also patch fields from this object. properties: apiVersion: default: kubernetes.crossplane.io/v1alpha1 description: APIVersion of the referenced object. type: string fieldPath: - description: FieldPath is the path of the field on the resource - whose value is to be used as input. + description: |- + FieldPath is the path of the field on the resource whose value is to be + used as input. type: string kind: default: Object @@ -371,22 +393,23 @@ spec: - name type: object toFieldPath: - description: ToFieldPath is the path of the field on the resource - whose value will be changed with the result of transforms. - Leave empty if you'd like to propagate to the same path as - patchesFrom.fieldPath. + description: |- + ToFieldPath is the path of the field on the resource whose value will + be changed with the result of transforms. Leave empty if you'd like to + propagate to the same path as patchesFrom.fieldPath. type: string type: object type: array writeConnectionSecretToRef: - description: WriteConnectionSecretToReference specifies the namespace - and name of a Secret to which any connection details for this managed - resource should be written. Connection details frequently include - the endpoint, username, and password required to connect to the - managed resource. This field is planned to be replaced in a future - release in favor of PublishConnectionDetailsTo. Currently, both - could be set independently and connection details would be published - to both without affecting each other. + description: |- + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + This field is planned to be replaced in a future release in favor of + PublishConnectionDetailsTo. Currently, both could be set independently + and connection details would be published to both without affecting + each other. properties: name: description: Name of the secret. @@ -419,13 +442,15 @@ spec: description: A Condition that may apply to a resource. properties: lastTransitionTime: - description: LastTransitionTime is the last time this condition - transitioned from one status to another. + description: |- + LastTransitionTime is the last time this condition transitioned from one + status to another. format: date-time type: string message: - description: A Message containing details about this condition's - last transition from one status to another, if any. + description: |- + A Message containing details about this condition's last transition from + one status to another, if any. type: string reason: description: A Reason for this condition's last transition from @@ -436,8 +461,9 @@ spec: False, or Unknown? type: string type: - description: Type of this condition. At most one of each condition - type may apply to a resource at any point in time. + description: |- + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. type: string required: - lastTransitionTime @@ -491,14 +517,19 @@ spec: description: A Object is an provider Kubernetes API type properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -514,48 +545,56 @@ spec: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string toConnectionSecretKey: type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object x-kubernetes-map-type: atomic type: array deletionPolicy: default: Delete - description: 'DeletionPolicy specifies what will happen to the underlying - external when this managed resource is deleted - either "Delete" - or "Orphan" the external resource. This field is planned to be deprecated - in favor of the ManagementPolicies field in a future release. Currently, - both could be set independently and non-default values would be - honored if the feature flag is enabled. See the design doc for more - information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223' + description: |- + DeletionPolicy specifies what will happen to the underlying external + when this managed resource is deleted - either "Delete" or "Orphan" the + external resource. + This field is planned to be deprecated in favor of the ManagementPolicies + field in a future release. Currently, both could be set independently and + non-default values would be honored if the feature flag is enabled. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 enum: - Orphan - Delete @@ -575,19 +614,21 @@ spec: managementPolicies: default: - '*' - description: 'THIS IS A BETA FIELD. It is on by default but can be - opted out through a Crossplane feature flag. ManagementPolicies - specify the array of actions Crossplane is allowed to take on the - managed and external resources. This field is planned to replace - the DeletionPolicy field in a future release. Currently, both could - be set independently and non-default values would be honored if - the feature flag is enabled. If both are custom, the DeletionPolicy - field will be ignored. See the design doc for more information: - https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 - and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md' + description: |- + THIS IS A BETA FIELD. It is on by default but can be opted out + through a Crossplane feature flag. + ManagementPolicies specify the array of actions Crossplane is allowed to + take on the managed and external resources. + This field is planned to replace the DeletionPolicy field in a future + release. Currently, both could be set independently and non-default + values would be honored if the feature flag is enabled. If both are + custom, the DeletionPolicy field will be ignored. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md items: - description: A ManagementAction represents an action that the Crossplane - controllers can take on an external resource. + description: |- + A ManagementAction represents an action that the Crossplane controllers + can take on an external resource. enum: - Observe - Create @@ -600,9 +641,10 @@ spec: providerConfigRef: default: name: default - description: ProviderConfigReference specifies how the provider that - will be used to create, observe, update, and delete this managed - resource should be configured. + description: |- + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. properties: name: description: Name of the referenced object. @@ -612,21 +654,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of this - reference is required. The default is 'Required', which - means the reconcile will fail if the reference cannot be - resolved. 'Optional' means this reference will be a no-op - if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should - be resolved. The default is 'IfNotPresent', which will attempt - to resolve the reference only when the corresponding field - is not present. Use 'Always' to resolve the reference on - every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent @@ -636,17 +678,19 @@ spec: - name type: object publishConnectionDetailsTo: - description: PublishConnectionDetailsTo specifies the connection secret - config which contains a name, metadata and a reference to secret - store config to which any connection details for this managed resource - should be written. Connection details frequently include the endpoint, - username, and password required to connect to the managed resource. + description: |- + PublishConnectionDetailsTo specifies the connection secret config which + contains a name, metadata and a reference to secret store config to + which any connection details for this managed resource should be written. + Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. properties: configRef: default: name: default - description: SecretStoreConfigRef specifies which secret store - config should be used for this ConnectionSecret. + description: |- + SecretStoreConfigRef specifies which secret store config should be used + for this ConnectionSecret. properties: name: description: Name of the referenced object. @@ -656,21 +700,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of - this reference is required. The default is 'Required', - which means the reconcile will fail if the reference - cannot be resolved. 'Optional' means this reference - will be a no-op if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should - be resolved. The default is 'IfNotPresent', which will - attempt to resolve the reference only when the corresponding - field is not present. Use 'Always' to resolve the reference - on every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent @@ -685,21 +729,22 @@ spec: annotations: additionalProperties: type: string - description: Annotations are the annotations to be added to - connection secret. - For Kubernetes secrets, this will be - used as "metadata.annotations". - It is up to Secret Store - implementation for others store types. + description: |- + Annotations are the annotations to be added to connection secret. + - For Kubernetes secrets, this will be used as "metadata.annotations". + - It is up to Secret Store implementation for others store types. type: object labels: additionalProperties: type: string - description: Labels are the labels/tags to be added to connection - secret. - For Kubernetes secrets, this will be used as "metadata.labels". - - It is up to Secret Store implementation for others store - types. + description: |- + Labels are the labels/tags to be added to connection secret. + - For Kubernetes secrets, this will be used as "metadata.labels". + - It is up to Secret Store implementation for others store types. type: object type: - description: Type is the SecretType for the connection secret. + description: |- + Type is the SecretType for the connection secret. - Only valid for Kubernetes Secret Stores. type: string type: object @@ -710,9 +755,10 @@ spec: - name type: object readiness: - description: Readiness defines how the object's readiness condition - should be computed, if not specified it will be considered ready - as soon as the underlying external resource is considered up-to-date. + description: |- + Readiness defines how the object's readiness condition should be computed, + if not specified it will be considered ready as soon as the underlying external + resource is considered up-to-date. properties: policy: default: SuccessfulCreate @@ -726,13 +772,14 @@ spec: type: object references: items: - description: Reference refers to an Object or arbitrary Kubernetes - resource and optionally patch values from that resource to the - current Object. + description: |- + Reference refers to an Object or arbitrary Kubernetes resource and optionally + patch values from that resource to the current Object. properties: dependsOn: - description: DependsOn is used to declare dependency on other - Object or arbitrary Kubernetes resource. + description: |- + DependsOn is used to declare dependency on other Object or arbitrary + Kubernetes resource. properties: apiVersion: default: kubernetes.crossplane.io/v1alpha1 @@ -752,17 +799,18 @@ spec: - name type: object patchesFrom: - description: PatchesFrom is used to declare dependency on other - Object or arbitrary Kubernetes resource, and also patch fields - from this object. + description: |- + PatchesFrom is used to declare dependency on other Object or arbitrary + Kubernetes resource, and also patch fields from this object. properties: apiVersion: default: kubernetes.crossplane.io/v1alpha1 description: APIVersion of the referenced object. type: string fieldPath: - description: FieldPath is the path of the field on the resource - whose value is to be used as input. + description: |- + FieldPath is the path of the field on the resource whose value is to be + used as input. type: string kind: default: Object @@ -779,22 +827,23 @@ spec: - name type: object toFieldPath: - description: ToFieldPath is the path of the field on the resource - whose value will be changed with the result of transforms. - Leave empty if you'd like to propagate to the same path as - patchesFrom.fieldPath. + description: |- + ToFieldPath is the path of the field on the resource whose value will + be changed with the result of transforms. Leave empty if you'd like to + propagate to the same path as patchesFrom.fieldPath. type: string type: object type: array writeConnectionSecretToRef: - description: WriteConnectionSecretToReference specifies the namespace - and name of a Secret to which any connection details for this managed - resource should be written. Connection details frequently include - the endpoint, username, and password required to connect to the - managed resource. This field is planned to be replaced in a future - release in favor of PublishConnectionDetailsTo. Currently, both - could be set independently and connection details would be published - to both without affecting each other. + description: |- + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + This field is planned to be replaced in a future release in favor of + PublishConnectionDetailsTo. Currently, both could be set independently + and connection details would be published to both without affecting + each other. properties: name: description: Name of the secret. @@ -827,13 +876,15 @@ spec: description: A Condition that may apply to a resource. properties: lastTransitionTime: - description: LastTransitionTime is the last time this condition - transitioned from one status to another. + description: |- + LastTransitionTime is the last time this condition transitioned from one + status to another. format: date-time type: string message: - description: A Message containing details about this condition's - last transition from one status to another, if any. + description: |- + A Message containing details about this condition's last transition from + one status to another, if any. type: string reason: description: A Reason for this condition's last transition from @@ -844,8 +895,9 @@ spec: False, or Unknown? type: string type: - description: Type of this condition. At most one of each condition - type may apply to a resource at any point in time. + description: |- + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. type: string required: - lastTransitionTime diff --git a/package/crds/kubernetes.crossplane.io_providerconfigs.yaml b/package/crds/kubernetes.crossplane.io_providerconfigs.yaml index 3cbd3d2d..ebd9df60 100644 --- a/package/crds/kubernetes.crossplane.io_providerconfigs.yaml +++ b/package/crds/kubernetes.crossplane.io_providerconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: providerconfigs.kubernetes.crossplane.io spec: group: kubernetes.crossplane.io @@ -32,14 +32,19 @@ spec: description: A ProviderConfig configures a Template provider. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -47,12 +52,14 @@ spec: description: A ProviderConfigSpec defines the desired state of a ProviderConfig. properties: credentials: - description: Credentials used to connect to the Kubernetes API. Typically - a kubeconfig file. Use InjectedIdentity for in-cluster config. + description: |- + Credentials used to connect to the Kubernetes API. Typically a + kubeconfig file. Use InjectedIdentity for in-cluster config. properties: env: - description: Env is a reference to an environment variable that - contains credentials that must be used to connect to the provider. + description: |- + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. properties: name: description: Name is the name of an environment variable. @@ -61,8 +68,9 @@ spec: - name type: object fs: - description: Fs is a reference to a filesystem location that contains - credentials that must be used to connect to the provider. + description: |- + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. properties: path: description: Path is a filesystem path. @@ -71,8 +79,9 @@ spec: - path type: object secretRef: - description: A SecretRef is a reference to a secret key that contains - the credentials that must be used to connect to the provider. + description: |- + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. properties: key: description: The key to select. @@ -101,13 +110,15 @@ spec: - source type: object identity: - description: Identity used to authenticate to the Kubernetes API. - The identity credentials can be used to supplement kubeconfig 'credentials', - for example by configuring a bearer token source such as OAuth. + description: |- + Identity used to authenticate to the Kubernetes API. The identity + credentials can be used to supplement kubeconfig 'credentials', for + example by configuring a bearer token source such as OAuth. properties: env: - description: Env is a reference to an environment variable that - contains credentials that must be used to connect to the provider. + description: |- + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. properties: name: description: Name is the name of an environment variable. @@ -116,8 +127,9 @@ spec: - name type: object fs: - description: Fs is a reference to a filesystem location that contains - credentials that must be used to connect to the provider. + description: |- + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. properties: path: description: Path is a filesystem path. @@ -126,8 +138,9 @@ spec: - path type: object secretRef: - description: A SecretRef is a reference to a secret key that contains - the credentials that must be used to connect to the provider. + description: |- + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. properties: key: description: The key to select. @@ -174,13 +187,15 @@ spec: description: A Condition that may apply to a resource. properties: lastTransitionTime: - description: LastTransitionTime is the last time this condition - transitioned from one status to another. + description: |- + LastTransitionTime is the last time this condition transitioned from one + status to another. format: date-time type: string message: - description: A Message containing details about this condition's - last transition from one status to another, if any. + description: |- + A Message containing details about this condition's last transition from + one status to another, if any. type: string reason: description: A Reason for this condition's last transition from @@ -191,8 +206,9 @@ spec: False, or Unknown? type: string type: - description: Type of this condition. At most one of each condition - type may apply to a resource at any point in time. + description: |- + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. type: string required: - lastTransitionTime diff --git a/package/crds/kubernetes.crossplane.io_providerconfigusages.yaml b/package/crds/kubernetes.crossplane.io_providerconfigusages.yaml index 0b69462f..9b27d532 100644 --- a/package/crds/kubernetes.crossplane.io_providerconfigusages.yaml +++ b/package/crds/kubernetes.crossplane.io_providerconfigusages.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: providerconfigusages.kubernetes.crossplane.io spec: group: kubernetes.crossplane.io @@ -37,14 +37,19 @@ spec: description: A ProviderConfigUsage indicates that a resource is using a ProviderConfig. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -59,19 +64,21 @@ spec: properties: resolution: default: Required - description: Resolution specifies whether resolution of this reference - is required. The default is 'Required', which means the reconcile - will fail if the reference cannot be resolved. 'Optional' means - this reference will be a no-op if it cannot be resolved. + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. enum: - Required - Optional type: string resolve: - description: Resolve specifies when this reference should be resolved. - The default is 'IfNotPresent', which will attempt to resolve - the reference only when the corresponding field is not present. - Use 'Always' to resolve the reference on every reconcile. + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. enum: - Always - IfNotPresent From 8592e3337aaebcc5ad4fddd29c6f9a47aaacac32 Mon Sep 17 00:00:00 2001 From: rladdukodiraghav Date: Wed, 7 Feb 2024 15:35:58 -0800 Subject: [PATCH 05/36] enable opt-in to redact Secret data from Object status Signed-off-by: rladdukodiraghav Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 3 +- internal/controller/kubernetes.go | 14 +++---- internal/controller/object/object.go | 50 ++++++++++++++++++++--- internal/controller/object/object_test.go | 39 ++++++++++++++++++ 4 files changed, 91 insertions(+), 15 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 7d077d2f..5f51bf9f 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -57,6 +57,7 @@ func main() { leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("10").Int() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() + enableSanitizeSecrets = app.Flag("enable-sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("ENABLE_SANITIZE_SECRETS").Bool() ) kingpin.MustParse(app.Parse(os.Args[1:])) @@ -129,7 +130,7 @@ func main() { // notice and remove when we drop support for v1alpha1. kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr).For(&v1alpha1.Object{}).Complete(), "Cannot create Object webhook") - kingpin.FatalIfError(object.Setup(mgr, o), "Cannot setup controller") + kingpin.FatalIfError(object.Setup(mgr, o, *enableSanitizeSecrets), "Cannot setup controller") kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") } diff --git a/internal/controller/kubernetes.go b/internal/controller/kubernetes.go index 0bba5271..951577bd 100644 --- a/internal/controller/kubernetes.go +++ b/internal/controller/kubernetes.go @@ -27,14 +27,12 @@ import ( // Setup creates all Template controllers with the supplied logger and adds them to // the supplied manager. -func Setup(mgr ctrl.Manager, o controller.Options) error { - for _, setup := range []func(ctrl.Manager, controller.Options) error{ - config.Setup, - object.Setup, - } { - if err := setup(mgr, o); err != nil { - return err - } +func Setup(mgr ctrl.Manager, o controller.Options, enableSanitizeSecrets bool) error { + if err := config.Setup(mgr, o); err != nil { + return err + } + if err := object.Setup(mgr, o, enableSanitizeSecrets); err != nil { + return err } return nil } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 82f25a13..cd1c595f 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" @@ -87,10 +88,11 @@ const ( errGetConnectionDetails = "cannot get connection details" errGetValueAtFieldPath = "cannot get value at fieldPath" errDecodeSecretData = "cannot decode secret data" + errSanitizeSecretData = "failed sanitizing secret data" ) // Setup adds a controller that reconciles Object managed resources. -func Setup(mgr ctrl.Manager, o controller.Options) error { +func Setup(mgr ctrl.Manager, o controller.Options, enableSantizeSecrets bool) error { name := managed.ControllerName(v1alpha2.ObjectGroupKind) cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} @@ -98,6 +100,7 @@ func Setup(mgr ctrl.Manager, o controller.Options) error { reconcilerOptions := []managed.ReconcilerOption{ managed.WithExternalConnecter(&connector{ logger: o.Logger, + santizeSecrets: enableSantizeSecrets, kube: mgr.GetClient(), usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), kcfgExtractorFn: resource.CommonCredentialExtractor, @@ -132,9 +135,10 @@ func Setup(mgr ctrl.Manager, o controller.Options) error { } type connector struct { - kube client.Client - usage resource.Tracker - logger logging.Logger + kube client.Client + usage resource.Tracker + logger logging.Logger + santizeSecrets bool kcfgExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) gcpExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) @@ -232,7 +236,8 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E Client: k, Applicator: resource.NewAPIPatchingApplicator(k), }, - localClient: c.kube, + localClient: c.kube, + sanitizeSecrets: c.santizeSecrets, }, nil } @@ -240,7 +245,8 @@ type external struct { logger logging.Logger client resource.ClientApplicator // localClient is specifically used to connect to local cluster - localClient client.Client + localClient client.Client + sanitizeSecrets bool } func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.ExternalObservation, error) { @@ -382,6 +388,16 @@ func getLastApplied(obj *v1alpha2.Object, observed *unstructured.Unstructured) ( func (c *external) setObserved(obj *v1alpha2.Object, observed *unstructured.Unstructured) error { var err error + + if c.sanitizeSecrets { + if observed.GetKind() == "Secret" && observed.GetAPIVersion() == "v1" { + observed, err = sanitizeUnstructuredSecret(observed) + if err != nil { + return errors.Wrap(err, errSanitizeSecretData) + } + } + } + if obj.Status.AtProvider.Manifest.Raw, err = observed.MarshalJSON(); err != nil { return errors.Wrap(err, errFailedToMarshalExisting) } @@ -675,3 +691,25 @@ func unstructuredFromObjectRef(r v1.ObjectReference) unstructured.Unstructured { return u } + +// sanitizeUnstructuredSecret redacts the data field of a Secret object +func sanitizeUnstructuredSecret(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + redactedUnstructured := &unstructured.Unstructured{} + s := &v1.Secret{} + + err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), s) + if err != nil { + return redactedUnstructured, fmt.Errorf("cannot convert unstructured to secret: %w", err) + } + + s.Data = map[string][]byte{"redacted": []byte(nil)} + + redactedObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(s) + if err != nil { + return redactedUnstructured, fmt.Errorf("cannot convert secret to unstructured: %w", err) + } + + redactedUnstructured.SetUnstructuredContent(redactedObj) + + return redactedUnstructured, nil +} diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 9b1e76e2..95906176 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -2056,3 +2056,42 @@ func Test_updateConditionFromObserved(t *testing.T) { }) } } + +func TestSanitizeUnstructuredSecret(t *testing.T) { + unstructuredSecret := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": "my-secret", + "namespace": "my-namespace", + }, + "type": "Opaque", + "data": map[string]interface{}{ + "key1": "dGVzdGluZwo=", + }, + }, + } + expected := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": "my-secret", + "namespace": "my-namespace", + "creationTimestamp": nil, + }, + "type": "Opaque", + "data": map[string]interface{}{ + "redacted": nil, + }, + }, + } + secret, err := sanitizeUnstructuredSecret(unstructuredSecret) + if err != nil { + t.Fatalf("sanitizeUnstructuredSecret(...): expected nil, got error: %s", err) + } + if diff := cmp.Diff(expected, secret); diff != "" { + t.Errorf("sanitizeUnstructuredSecret(...): -want result, +got result: %s", diff) + } +} From d29c762407c5b7e00203b61727dbe0c67d6d685c Mon Sep 17 00:00:00 2001 From: rladdukodiraghav Date: Thu, 8 Feb 2024 18:49:06 -0800 Subject: [PATCH 06/36] address review comments: use fieldPath library to set redacted data Signed-off-by: rladdukodiraghav Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 4 +-- internal/controller/kubernetes.go | 4 +-- internal/controller/object/object.go | 43 ++++++----------------- internal/controller/object/object_test.go | 39 -------------------- 4 files changed, 14 insertions(+), 76 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 5f51bf9f..cb764550 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -57,7 +57,7 @@ func main() { leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("10").Int() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() - enableSanitizeSecrets = app.Flag("enable-sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("ENABLE_SANITIZE_SECRETS").Bool() + sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() ) kingpin.MustParse(app.Parse(os.Args[1:])) @@ -130,7 +130,7 @@ func main() { // notice and remove when we drop support for v1alpha1. kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr).For(&v1alpha1.Object{}).Complete(), "Cannot create Object webhook") - kingpin.FatalIfError(object.Setup(mgr, o, *enableSanitizeSecrets), "Cannot setup controller") + kingpin.FatalIfError(object.Setup(mgr, o, *sanitizeSecrets), "Cannot setup controller") kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") } diff --git a/internal/controller/kubernetes.go b/internal/controller/kubernetes.go index 951577bd..7b957c61 100644 --- a/internal/controller/kubernetes.go +++ b/internal/controller/kubernetes.go @@ -27,11 +27,11 @@ import ( // Setup creates all Template controllers with the supplied logger and adds them to // the supplied manager. -func Setup(mgr ctrl.Manager, o controller.Options, enableSanitizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { if err := config.Setup(mgr, o); err != nil { return err } - if err := object.Setup(mgr, o, enableSanitizeSecrets); err != nil { + if err := object.Setup(mgr, o, sanitizeSecrets); err != nil { return err } return nil diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index cd1c595f..dc054ebc 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/api/equality" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" @@ -88,11 +87,11 @@ const ( errGetConnectionDetails = "cannot get connection details" errGetValueAtFieldPath = "cannot get value at fieldPath" errDecodeSecretData = "cannot decode secret data" - errSanitizeSecretData = "failed sanitizing secret data" + errSanitizeSecretData = "cannot sanitize secret data" ) // Setup adds a controller that reconciles Object managed resources. -func Setup(mgr ctrl.Manager, o controller.Options, enableSantizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { name := managed.ControllerName(v1alpha2.ObjectGroupKind) cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} @@ -100,7 +99,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, enableSantizeSecrets bool) er reconcilerOptions := []managed.ReconcilerOption{ managed.WithExternalConnecter(&connector{ logger: o.Logger, - santizeSecrets: enableSantizeSecrets, + sanitizeSecrets: sanitizeSecrets, kube: mgr.GetClient(), usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), kcfgExtractorFn: resource.CommonCredentialExtractor, @@ -135,10 +134,10 @@ func Setup(mgr ctrl.Manager, o controller.Options, enableSantizeSecrets bool) er } type connector struct { - kube client.Client - usage resource.Tracker - logger logging.Logger - santizeSecrets bool + kube client.Client + usage resource.Tracker + logger logging.Logger + sanitizeSecrets bool kcfgExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) gcpExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) @@ -237,7 +236,7 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E Applicator: resource.NewAPIPatchingApplicator(k), }, localClient: c.kube, - sanitizeSecrets: c.santizeSecrets, + sanitizeSecrets: c.sanitizeSecrets, }, nil } @@ -391,8 +390,8 @@ func (c *external) setObserved(obj *v1alpha2.Object, observed *unstructured.Unst if c.sanitizeSecrets { if observed.GetKind() == "Secret" && observed.GetAPIVersion() == "v1" { - observed, err = sanitizeUnstructuredSecret(observed) - if err != nil { + data := map[string][]byte{"redacted": []byte(nil)} + if err = fieldpath.Pave(observed.Object).SetValue("data", data); err != nil { return errors.Wrap(err, errSanitizeSecretData) } } @@ -691,25 +690,3 @@ func unstructuredFromObjectRef(r v1.ObjectReference) unstructured.Unstructured { return u } - -// sanitizeUnstructuredSecret redacts the data field of a Secret object -func sanitizeUnstructuredSecret(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { - redactedUnstructured := &unstructured.Unstructured{} - s := &v1.Secret{} - - err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), s) - if err != nil { - return redactedUnstructured, fmt.Errorf("cannot convert unstructured to secret: %w", err) - } - - s.Data = map[string][]byte{"redacted": []byte(nil)} - - redactedObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(s) - if err != nil { - return redactedUnstructured, fmt.Errorf("cannot convert secret to unstructured: %w", err) - } - - redactedUnstructured.SetUnstructuredContent(redactedObj) - - return redactedUnstructured, nil -} diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 95906176..9b1e76e2 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -2056,42 +2056,3 @@ func Test_updateConditionFromObserved(t *testing.T) { }) } } - -func TestSanitizeUnstructuredSecret(t *testing.T) { - unstructuredSecret := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Secret", - "metadata": map[string]interface{}{ - "name": "my-secret", - "namespace": "my-namespace", - }, - "type": "Opaque", - "data": map[string]interface{}{ - "key1": "dGVzdGluZwo=", - }, - }, - } - expected := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "v1", - "kind": "Secret", - "metadata": map[string]interface{}{ - "name": "my-secret", - "namespace": "my-namespace", - "creationTimestamp": nil, - }, - "type": "Opaque", - "data": map[string]interface{}{ - "redacted": nil, - }, - }, - } - secret, err := sanitizeUnstructuredSecret(unstructuredSecret) - if err != nil { - t.Fatalf("sanitizeUnstructuredSecret(...): expected nil, got error: %s", err) - } - if diff := cmp.Diff(expected, secret); diff != "" { - t.Errorf("sanitizeUnstructuredSecret(...): -want result, +got result: %s", diff) - } -} From 292c7b5bf27b0da336e6c2069b88631f317b5e53 Mon Sep 17 00:00:00 2001 From: rladdukodiraghav Date: Fri, 9 Feb 2024 07:55:37 -0800 Subject: [PATCH 07/36] update CI go version to go-1.21 Signed-off-by: rladdukodiraghav Signed-off-by: Alexander Brandstedt --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24060d14..e81a0bf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: env: # Common versions - GO_VERSION: '1.20.12' + GO_VERSION: '1.21.7' GOLANGCI_VERSION: 'v1.55.2' DOCKER_BUILDX_VERSION: 'v0.8.2' From c596239aa4c10290cdeeb5cd3fcb1c041ebaf225 Mon Sep 17 00:00:00 2001 From: ravilr Date: Tue, 13 Feb 2024 14:07:54 -0800 Subject: [PATCH 08/36] Add jitter to poll interval Signed-off-by: ravilr Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 10 +++++++++- internal/controller/kubernetes.go | 6 ++++-- internal/controller/object/object.go | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index cb764550..8e6bf113 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -72,6 +72,14 @@ func main() { ctrl.SetLogger(zl) } + // configure the jitter to be the 10% of the poll interval + pollJitter := time.Duration(float64(*pollInterval) * 0.1) + log.Debug("Starting", + "sync-interval", syncInterval.String(), + "poll-interval", pollInterval.String(), + "poll-jitter", pollJitter.String(), + "max-reconcile-rate", *maxReconcileRate) + cfg, err := ctrl.GetConfig() kingpin.FatalIfError(err, "Cannot get API server rest config") @@ -130,7 +138,7 @@ func main() { // notice and remove when we drop support for v1alpha1. kingpin.FatalIfError(ctrl.NewWebhookManagedBy(mgr).For(&v1alpha1.Object{}).Complete(), "Cannot create Object webhook") - kingpin.FatalIfError(object.Setup(mgr, o, *sanitizeSecrets), "Cannot setup controller") + kingpin.FatalIfError(object.Setup(mgr, o, *sanitizeSecrets, pollJitter), "Cannot setup controller") kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager") } diff --git a/internal/controller/kubernetes.go b/internal/controller/kubernetes.go index 7b957c61..5fb3bf49 100644 --- a/internal/controller/kubernetes.go +++ b/internal/controller/kubernetes.go @@ -17,6 +17,8 @@ limitations under the License. package controller import ( + "time" + ctrl "sigs.k8s.io/controller-runtime" "github.com/crossplane/crossplane-runtime/pkg/controller" @@ -27,11 +29,11 @@ import ( // Setup creates all Template controllers with the supplied logger and adds them to // the supplied manager. -func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJitter time.Duration) error { if err := config.Setup(mgr, o); err != nil { return err } - if err := object.Setup(mgr, o, sanitizeSecrets); err != nil { + if err := object.Setup(mgr, o, sanitizeSecrets, pollJitter); err != nil { return err } return nil diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index dc054ebc..a160872c 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "strings" + "time" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" @@ -91,7 +92,7 @@ const ( ) // Setup adds a controller that reconciles Object managed resources. -func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { +func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJitter time.Duration) error { name := managed.ControllerName(v1alpha2.ObjectGroupKind) cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} @@ -112,6 +113,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool) error { }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), + managed.WithPollJitterHook(pollJitter), managed.WithLogger(o.Logger.WithValues("controller", name)), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithConnectionPublishers(cps...), From 90c5b60235fb47a4eb88998b73f4b6bf3a9df77a Mon Sep 17 00:00:00 2001 From: ravilr Date: Thu, 15 Feb 2024 19:16:09 -0800 Subject: [PATCH 09/36] make poll jitter percentage configurable through flag option Signed-off-by: ravilr Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 8e6bf113..18019c12 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -54,6 +54,7 @@ func main() { debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration() + pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("10").Int() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() @@ -72,8 +73,10 @@ func main() { ctrl.SetLogger(zl) } - // configure the jitter to be the 10% of the poll interval - pollJitter := time.Duration(float64(*pollInterval) * 0.1) + if *pollJitterPercentage >= 100 { + kingpin.Fatalf("invalid --poll-jitter-percentage %v must be less than 100", *pollJitterPercentage) + } + pollJitter := time.Duration(float64(*pollInterval) * (float64(*pollJitterPercentage) / 100.0)) log.Debug("Starting", "sync-interval", syncInterval.String(), "poll-interval", pollInterval.String(), From a68bf67b8338ae08064c270cf0281749292642bd Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Mon, 19 Feb 2024 15:35:33 +0300 Subject: [PATCH 10/36] Use better defaults for poll interval and max reconcile rate Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 18019c12..ab235cc7 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -53,10 +53,10 @@ func main() { app = kingpin.New(filepath.Base(os.Args[0]), "Template support for Crossplane.").DefaultEnvars() debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() - pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("1m").Duration() + pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration() pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() - maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("10").Int() + maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("100").Int() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() ) From fddaba8960c355e65fc6f6e1fe4d9e7c943da764 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Tue, 20 Feb 2024 21:44:33 +0300 Subject: [PATCH 11/36] Do not wait for poll interval if not ready Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/object.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index a160872c..1347ac5b 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "math/rand" "strings" "time" @@ -113,7 +114,15 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), - managed.WithPollJitterHook(pollJitter), + managed.WithPollIntervalHook(func(mg resource.Managed, pollInterval time.Duration) time.Duration { + if mg.GetCondition(xpv1.TypeReady).Status != v1.ConditionTrue { + // If the resource is not ready, we should poll more frequently not to delay time to readiness. + pollInterval = 30 * time.Second + } + // This is the same as runtime default poll interval with jitter, see: + // https://github.com/crossplane/crossplane-runtime/blob/7fcb8c5cad6fc4abb6649813b92ab92e1832d368/pkg/reconciler/managed/reconciler.go#L573 + return pollInterval + time.Duration((rand.Float64()-0.5)*2*float64(pollJitter)) //nolint G404 // No need for secure randomness + }), managed.WithLogger(o.Logger.WithValues("controller", name)), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithConnectionPublishers(cps...), From 382c96b903bacfe396ae5c35ca5b2b63dc24b7f6 Mon Sep 17 00:00:00 2001 From: ravilr Date: Fri, 23 Feb 2024 16:20:01 -0800 Subject: [PATCH 12/36] fix missing ToConnectionSecretKey in conversion Signed-off-by: ravilr Signed-off-by: Alexander Brandstedt --- apis/object/v1alpha1/conversion.go | 6 ++-- apis/object/v1alpha1/conversion_test.go | 16 +++++++++ .../object/deprecated/connection-details.yaml | 33 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 examples/object/deprecated/connection-details.yaml diff --git a/apis/object/v1alpha1/conversion.go b/apis/object/v1alpha1/conversion.go index a1f5cef9..33f3e401 100644 --- a/apis/object/v1alpha1/conversion.go +++ b/apis/object/v1alpha1/conversion.go @@ -43,7 +43,8 @@ func (src *Object) ConvertTo(dstRaw conversion.Hub) error { // nolint:golint // connectionDetails := []v1alpha2.ConnectionDetail{} for _, cd := range src.Spec.ConnectionDetails { connectionDetails = append(connectionDetails, v1alpha2.ConnectionDetail{ - ObjectReference: cd.ObjectReference, + ObjectReference: cd.ObjectReference, + ToConnectionSecretKey: cd.ToConnectionSecretKey, }) } @@ -123,7 +124,8 @@ func (dst *Object) ConvertFrom(srcRaw conversion.Hub) error { // nolint:golint, connectionDetails := []ConnectionDetail{} for _, cd := range src.Spec.ConnectionDetails { connectionDetails = append(connectionDetails, ConnectionDetail{ - ObjectReference: cd.ObjectReference, + ObjectReference: cd.ObjectReference, + ToConnectionSecretKey: cd.ToConnectionSecretKey, }) } diff --git a/apis/object/v1alpha1/conversion_test.go b/apis/object/v1alpha1/conversion_test.go index 122a93fe..dd47d8c4 100644 --- a/apis/object/v1alpha1/conversion_test.go +++ b/apis/object/v1alpha1/conversion_test.go @@ -64,7 +64,9 @@ func TestConvertTo(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha1.ObjectParameters{ @@ -111,7 +113,9 @@ func TestConvertTo(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha2.ObjectParameters{ @@ -196,7 +200,9 @@ func TestConvertTo(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha1.ObjectParameters{ @@ -229,7 +235,9 @@ func TestConvertTo(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha2.ObjectParameters{ @@ -312,7 +320,9 @@ func TestConvertFrom(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha2.ObjectParameters{ @@ -357,7 +367,9 @@ func TestConvertFrom(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha1.ObjectParameters{ @@ -407,7 +419,9 @@ func TestConvertFrom(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha2.ObjectParameters{ @@ -438,7 +452,9 @@ func TestConvertFrom(t *testing.T) { APIVersion: "v1", Kind: "Secret", Name: "topsecret", + FieldPath: "data.token", }, + ToConnectionSecretKey: "token", }, }, ForProvider: v1alpha1.ObjectParameters{ diff --git a/examples/object/deprecated/connection-details.yaml b/examples/object/deprecated/connection-details.yaml new file mode 100644 index 00000000..255d45f0 --- /dev/null +++ b/examples/object/deprecated/connection-details.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: default + name: test-sa +--- +apiVersion: kubernetes.crossplane.io/v1alpha1 +kind: Object +metadata: + name: test-sa +spec: + connectionDetails: + - apiVersion: v1 + fieldPath: data.token + kind: Secret + name: test-sa-token + namespace: default + toConnectionSecretKey: token + forProvider: + manifest: + apiVersion: v1 + kind: Secret + metadata: + annotations: + kubernetes.io/service-account.name: test-sa + name: test-sa-token + namespace: default + type: kubernetes.io/service-account-token + providerConfigRef: + name: kubernetes-provider + writeConnectionSecretToRef: + name: test-sa-conn-out + namespace: default From 644862402a34a1923fbe02e5982aaa8620d19589 Mon Sep 17 00:00:00 2001 From: Predrag Knezevic Date: Mon, 8 Apr 2024 10:59:36 +0200 Subject: [PATCH 13/36] Add `ObservedObjectCollection` API type (#217) * Add `ObservedObjectCollection` API type Objects in the collection are defined by: * GVK * optional namespace * label selector The objects are fetched using the specified provider config and for the matched objects the provider creates counterpart observe-only objects in the local cluster. The created objects are owned by the collection resource and reconciled as usual by the provider. They are labeled with a common label, so that they can be fetched easily. The label is discoverable by reading `.status.membershipLabel` field of `ObservedObjectCollection`. Signed-off-by: Predrag Knezevic Signed-off-by: Alexander Brandstedt --- apis/kubernetes.go | 2 + apis/observedobjectcollection/v1alpha1/doc.go | 22 + .../v1alpha1/register.go | 50 +++ .../v1alpha1/types.go | 126 ++++++ .../v1alpha1/zz_generated.deepcopy.go | 205 +++++++++ examples/collection/collection.yaml | 42 ++ go.mod | 2 +- internal/clients/client.go | 88 ++++ internal/controller/kubernetes.go | 4 + internal/controller/object/object.go | 102 +---- internal/controller/object/object_test.go | 394 +---------------- .../observedobjectcollection/reconciler.go | 286 +++++++++++++ .../reconciler_test.go | 402 ++++++++++++++++++ ...ossplane.io_observedobjectcollections.yaml | 257 +++++++++++ 14 files changed, 1509 insertions(+), 473 deletions(-) create mode 100644 apis/observedobjectcollection/v1alpha1/doc.go create mode 100644 apis/observedobjectcollection/v1alpha1/register.go create mode 100644 apis/observedobjectcollection/v1alpha1/types.go create mode 100644 apis/observedobjectcollection/v1alpha1/zz_generated.deepcopy.go create mode 100644 examples/collection/collection.yaml create mode 100644 internal/controller/observedobjectcollection/reconciler.go create mode 100644 internal/controller/observedobjectcollection/reconciler_test.go create mode 100644 package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml diff --git a/apis/kubernetes.go b/apis/kubernetes.go index f49b7515..735ad61b 100644 --- a/apis/kubernetes.go +++ b/apis/kubernetes.go @@ -22,6 +22,7 @@ import ( objectv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha1" objectv1alhpa2 "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + observedobjectcollectionv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" templatev1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" ) @@ -31,6 +32,7 @@ func init() { templatev1alpha1.SchemeBuilder.AddToScheme, objectv1alpha1.SchemeBuilder.AddToScheme, objectv1alhpa2.SchemeBuilder.AddToScheme, + observedobjectcollectionv1alpha1.SchemeBuilder.AddToScheme, ) } diff --git a/apis/observedobjectcollection/v1alpha1/doc.go b/apis/observedobjectcollection/v1alpha1/doc.go new file mode 100644 index 00000000..b8e60b1d --- /dev/null +++ b/apis/observedobjectcollection/v1alpha1/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains the v1alpha1 group ObservedObjectCollection resources of the Kubernetes provider. +// +kubebuilder:ac:generate=true +// +kubebuilder:object:generate=true +// +groupName=kubernetes.crossplane.io +// +versionName=v1alpha1 +package v1alpha1 diff --git a/apis/observedobjectcollection/v1alpha1/register.go b/apis/observedobjectcollection/v1alpha1/register.go new file mode 100644 index 00000000..c438e3f9 --- /dev/null +++ b/apis/observedobjectcollection/v1alpha1/register.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "reflect" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +// Package type metadata. +const ( + Group = "kubernetes.crossplane.io" + Version = "v1alpha1" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +) + +// ProviderConfig type metadata. +var ( + ObservedObjectCollectionKind = reflect.TypeOf(ObservedObjectCollection{}).Name() + ObservedObjectCollectionGroupKind = schema.GroupKind{Group: Group, Kind: ObservedObjectCollectionKind}.String() + ObservedObjectCollectionAPIVersion = ObservedObjectCollectionKind + "." + SchemeGroupVersion.String() + ObservedObjectCollectionGroupVersionKind = SchemeGroupVersion.WithKind(ObservedObjectCollectionKind) +) + +func init() { + SchemeBuilder.Register(&ObservedObjectCollection{}, &ObservedObjectCollectionList{}) +} diff --git a/apis/observedobjectcollection/v1alpha1/types.go b/apis/observedobjectcollection/v1alpha1/types.go new file mode 100644 index 00000000..3ac93c4a --- /dev/null +++ b/apis/observedobjectcollection/v1alpha1/types.go @@ -0,0 +1,126 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + + v12 "github.com/crossplane/crossplane-runtime/apis/common/v1" +) + +// +kubebuilder:object:root=true + +// A ObservedObjectCollection is a provider Kubernetes API type +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="KIND",type="string",JSONPath=".spec.kind" +// +kubebuilder:printcolumn:name="APIVERSION",type="string",JSONPath=".spec.apiVersion",priority=1 +// +kubebuilder:printcolumn:name="PROVIDERCONFIG",type="string",JSONPath=".spec.providerConfigRef.name" +// +kubebuilder:printcolumn:name="SYNCED",type="string",JSONPath=".status.conditions[?(@.type=='Synced')].status" +// +kubebuilder:printcolumn:name="READY",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:resource:scope=Cluster,categories={crossplane,managed,kubernetes} +// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) < 64",message="metadata.name max length is 63" +type ObservedObjectCollection struct { + v1.TypeMeta `json:",inline"` + v1.ObjectMeta `json:"metadata,omitempty"` + Spec ObservedObjectCollectionSpec `json:"spec"` + Status ObservedObjectCollectionStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ObservedObjectCollectionList contains a list of ObservedObjectCollection +type ObservedObjectCollectionList struct { + v1.TypeMeta `json:",inline"` + v1.ListMeta `json:"metadata,omitempty"` + Items []ObservedObjectCollection `json:"items"` +} + +// ObservedObjectCollectionSpec defines the desired state of ObservedObjectCollection +type ObservedObjectCollectionSpec struct { + + // ObserveObjects declares what criteria object need to fulfil + // to become a member of this collection + ObserveObjects ObserveObjectCriteria `json:"observeObjects"` + + // ProviderConfigReference specifies how the provider that will be used to + // create, observe, update, and delete this managed resource should be + // configured. + // +kubebuilder:default={"name": "default"} + ProviderConfigReference v12.Reference `json:"providerConfigRef,omitempty"` + + // Template when defined is used for creating Object instances + // +optional + Template *ObservedObjectTemplate `json:"objectTemplate,omitempty"` +} + +// ObserveObjectCriteria declares criteria for an object to be a part of collection +type ObserveObjectCriteria struct { + + // APIVersion of objects that should be matched by the selector + // +kubebuilder:validation:MinLength:=1 + APIVersion string `json:"apiVersion"` + + // Kind of objects that should be matched by the selector + // +kubebuilder:validation:MinLength:=1 + Kind string `json:"kind"` + + // Namespace where to look for objects. + // If omitted, search is performed across all namespaces. + // For cluster-scoped objects, omit it. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Selector defines the criteria for including objects into the collection + Selector v1.LabelSelector `json:"selector"` +} + +// ObservedObjectTemplate represents template used when creating observe-only Objects matching the given selector +type ObservedObjectTemplate struct { + + // Objects metadata + Metadata ObservedObjectTemplateMetadata `json:"metadata,omitempty"` +} + +// ObservedObjectTemplateMetadata represents objects metadata +type ObservedObjectTemplateMetadata struct { + + // Labels of an object + Labels map[string]string `json:"labels,omitempty"` + + // Annotations of an object + Annotations map[string]string `json:"annotations,omitempty"` +} + +// ObservedObjectCollectionStatus represents the observed state of a ObservedObjectCollection +type ObservedObjectCollectionStatus struct { + v12.ResourceStatus `json:",inline"` + + // MembershipLabel is the label set on each member of this collection + // and can be used for fetching them. + // +optional + MembershipLabel map[string]string `json:"membershipLabel,omitempty"` +} + +// ObservedObjectReference represents a reference to Object with ObserveOnly management policy +type ObservedObjectReference struct { + + // Name of the observed object + // +kubebuilder:validation:MinLength:=1 + // +kubebuilder:validation:MaxLength:=253 + Name string `json:"name"` +} diff --git a/apis/observedobjectcollection/v1alpha1/zz_generated.deepcopy.go b/apis/observedobjectcollection/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..76572f75 --- /dev/null +++ b/apis/observedobjectcollection/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,205 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2020 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObserveObjectCriteria) DeepCopyInto(out *ObserveObjectCriteria) { + *out = *in + in.Selector.DeepCopyInto(&out.Selector) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObserveObjectCriteria. +func (in *ObserveObjectCriteria) DeepCopy() *ObserveObjectCriteria { + if in == nil { + return nil + } + out := new(ObserveObjectCriteria) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectCollection) DeepCopyInto(out *ObservedObjectCollection) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectCollection. +func (in *ObservedObjectCollection) DeepCopy() *ObservedObjectCollection { + if in == nil { + return nil + } + out := new(ObservedObjectCollection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ObservedObjectCollection) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectCollectionList) DeepCopyInto(out *ObservedObjectCollectionList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ObservedObjectCollection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectCollectionList. +func (in *ObservedObjectCollectionList) DeepCopy() *ObservedObjectCollectionList { + if in == nil { + return nil + } + out := new(ObservedObjectCollectionList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ObservedObjectCollectionList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectCollectionSpec) DeepCopyInto(out *ObservedObjectCollectionSpec) { + *out = *in + in.ObserveObjects.DeepCopyInto(&out.ObserveObjects) + in.ProviderConfigReference.DeepCopyInto(&out.ProviderConfigReference) + if in.Template != nil { + in, out := &in.Template, &out.Template + *out = new(ObservedObjectTemplate) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectCollectionSpec. +func (in *ObservedObjectCollectionSpec) DeepCopy() *ObservedObjectCollectionSpec { + if in == nil { + return nil + } + out := new(ObservedObjectCollectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectCollectionStatus) DeepCopyInto(out *ObservedObjectCollectionStatus) { + *out = *in + in.ResourceStatus.DeepCopyInto(&out.ResourceStatus) + if in.MembershipLabel != nil { + in, out := &in.MembershipLabel, &out.MembershipLabel + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectCollectionStatus. +func (in *ObservedObjectCollectionStatus) DeepCopy() *ObservedObjectCollectionStatus { + if in == nil { + return nil + } + out := new(ObservedObjectCollectionStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectReference) DeepCopyInto(out *ObservedObjectReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectReference. +func (in *ObservedObjectReference) DeepCopy() *ObservedObjectReference { + if in == nil { + return nil + } + out := new(ObservedObjectReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectTemplate) DeepCopyInto(out *ObservedObjectTemplate) { + *out = *in + in.Metadata.DeepCopyInto(&out.Metadata) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectTemplate. +func (in *ObservedObjectTemplate) DeepCopy() *ObservedObjectTemplate { + if in == nil { + return nil + } + out := new(ObservedObjectTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObservedObjectTemplateMetadata) DeepCopyInto(out *ObservedObjectTemplateMetadata) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservedObjectTemplateMetadata. +func (in *ObservedObjectTemplateMetadata) DeepCopy() *ObservedObjectTemplateMetadata { + if in == nil { + return nil + } + out := new(ObservedObjectTemplateMetadata) + in.DeepCopyInto(out) + return out +} diff --git a/examples/collection/collection.yaml b/examples/collection/collection.yaml new file mode 100644 index 00000000..17c92c29 --- /dev/null +++ b/examples/collection/collection.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: configmap-bar + namespace: default + labels: + foo: bar +data: + sample-key: "sample-value" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: configmap-foo + namespace: default + labels: + foo: bar +data: + sample-key: "sample-value2" +--- +apiVersion: kubernetes.crossplane.io/v1alpha1 +kind: ObservedObjectCollection +metadata: + name: foo-collection +spec: + observeObjects: + apiVersion: v1 + kind: ConfigMap + selector: + matchLabels: + foo: bar + objectTemplate: + metadata: + labels: + l1: v1 + annotations: + a1: v1 + providerConfigRef: + name: kubernetes-provider + diff --git a/go.mod b/go.mod index 4ddb2dbb..aef6678d 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/crossplane/crossplane-runtime v1.15.0-rc.1 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.4.0 github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.26.0 @@ -58,7 +59,6 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/internal/clients/client.go b/internal/clients/client.go index 8360b580..eff1a1a0 100644 --- a/internal/clients/client.go +++ b/internal/clients/client.go @@ -14,11 +14,31 @@ limitations under the License. package clients import ( + "context" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/controller-runtime/pkg/client" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/resource" + + "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/azure" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/gke" +) + +const ( + errGetPC = "cannot get ProviderConfig" + errGetCreds = "cannot get credentials" + errCreateRestConfig = "cannot create new REST config using provider secret" + errExtractGoogleCredentials = "cannot extract Google Application Credentials" + errInjectGoogleCredentials = "cannot wrap REST client with Google Application Credentials" + errExtractAzureCredentials = "failed to extract Azure Application Credentials" + errInjectAzureCredentials = "failed to wrap REST client with Azure Application Credentials" ) // NewRESTConfig returns a rest config given a secret with connection information. @@ -86,3 +106,71 @@ func restConfigFromAPIConfig(c *api.Config) (*rest.Config, error) { return config, nil } + +// ClientForProvider returns the client for the given provider config +func ClientForProvider(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { //nolint:gocyclo + pc := &v1alpha1.ProviderConfig{} + if err := inclusterClient.Get(ctx, types.NamespacedName{Name: providerConfigName}, pc); err != nil { + return nil, errors.Wrap(err, errGetPC) + } + + var rc *rest.Config + var err error + + switch cd := pc.Spec.Credentials; cd.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + rc, err = rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, errCreateRestConfig) + } + default: + kc, err := resource.CommonCredentialExtractor(ctx, cd.Source, inclusterClient, cd.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errGetCreds) + } + + if rc, err = NewRESTConfig(kc); err != nil { + return nil, errors.Wrap(err, errCreateRestConfig) + } + } + + if id := pc.Spec.Identity; id != nil { + switch id.Type { + case v1alpha1.IdentityTypeGoogleApplicationCredentials: + switch id.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + if err := gke.WrapRESTConfig(ctx, rc, nil, gke.DefaultScopes...); err != nil { + return nil, errors.Wrap(err, errInjectGoogleCredentials) + } + default: + creds, err := resource.CommonCredentialExtractor(ctx, id.Source, inclusterClient, id.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errExtractGoogleCredentials) + } + + if err := gke.WrapRESTConfig(ctx, rc, creds, gke.DefaultScopes...); err != nil { + return nil, errors.Wrap(err, errInjectGoogleCredentials) + } + } + case v1alpha1.IdentityTypeAzureServicePrincipalCredentials: + switch id.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + return nil, errors.Errorf("%s is not supported as identity source for identity type %s", + xpv1.CredentialsSourceInjectedIdentity, v1alpha1.IdentityTypeAzureServicePrincipalCredentials) + default: + creds, err := resource.CommonCredentialExtractor(ctx, id.Source, inclusterClient, id.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errExtractAzureCredentials) + } + + if err := azure.WrapRESTConfig(ctx, rc, creds); err != nil { + return nil, errors.Wrap(err, errInjectAzureCredentials) + } + } + default: + return nil, errors.Errorf("unknown identity type: %s", id.Type) + } + } + + return NewKubeClient(rc) +} diff --git a/internal/controller/kubernetes.go b/internal/controller/kubernetes.go index 5fb3bf49..d5601332 100644 --- a/internal/controller/kubernetes.go +++ b/internal/controller/kubernetes.go @@ -25,6 +25,7 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/internal/controller/config" "github.com/crossplane-contrib/provider-kubernetes/internal/controller/object" + "github.com/crossplane-contrib/provider-kubernetes/internal/controller/observedobjectcollection" ) // Setup creates all Template controllers with the supplied logger and adds them to @@ -36,5 +37,8 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit if err := object.Setup(mgr, o, sanitizeSecrets, pollJitter); err != nil { return err } + if err := observedobjectcollection.Setup(mgr, o, pollJitter); err != nil { + return err + } return nil } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 1347ac5b..b1fd63f6 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -32,7 +32,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -50,26 +49,17 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" apisv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" "github.com/crossplane-contrib/provider-kubernetes/internal/clients" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients/azure" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients/gke" ) const ( errTrackPCUsage = "cannot track ProviderConfig usage" - errGetPC = "cannot get ProviderConfig" - errGetCreds = "cannot get credentials" errGetObject = "cannot get object" errCreateObject = "cannot create object" errApplyObject = "cannot apply object" errDeleteObject = "cannot delete object" - errNotKubernetesObject = "managed resource is not an Object custom resource" - errNewKubernetesClient = "cannot create new Kubernetes client" - errFailedToCreateRestConfig = "cannot create new REST config using provider secret" - errFailedToExtractGoogleCredentials = "cannot extract Google Application Credentials" - errFailedToInjectGoogleCredentials = "cannot wrap REST client with Google Application Credentials" - errFailedToExtractAzureCredentials = "failed to extract Azure Application Credentials" - errFailedToInjectAzureCredentials = "failed to wrap REST client with Azure Application Credentials" + errNotKubernetesObject = "managed resource is not an Object custom resource" + errNewKubernetesClient = "cannot create new Kubernetes client" errGetLastApplied = "cannot get last applied" errUnmarshalTemplate = "cannot unmarshal template" @@ -100,17 +90,11 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit reconcilerOptions := []managed.ReconcilerOption{ managed.WithExternalConnecter(&connector{ - logger: o.Logger, - sanitizeSecrets: sanitizeSecrets, - kube: mgr.GetClient(), - usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), - kcfgExtractorFn: resource.CommonCredentialExtractor, - gcpExtractorFn: resource.CommonCredentialExtractor, - gcpInjectorFn: gke.WrapRESTConfig, - azureExtractorFn: resource.CommonCredentialExtractor, - azureInjectorFn: azure.WrapRESTConfig, - newRESTConfigFn: clients.NewRESTConfig, - newKubeClientFn: clients.NewKubeClient, + logger: o.Logger, + sanitizeSecrets: sanitizeSecrets, + kube: mgr.GetClient(), + usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), + clientForProviderFn: clients.ClientForProvider, }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), @@ -150,13 +134,7 @@ type connector struct { logger logging.Logger sanitizeSecrets bool - kcfgExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - gcpExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - gcpInjectorFn func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error - azureExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - azureInjectorFn func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error - newRESTConfigFn func(kubeconfig []byte) (*rest.Config, error) - newKubeClientFn func(config *rest.Config) (client.Client, error) + clientForProviderFn func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) } func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.ExternalClient, error) { //nolint:gocyclo @@ -172,70 +150,8 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E return nil, errors.Wrap(err, errTrackPCUsage) } - pc := &apisv1alpha1.ProviderConfig{} - if err := c.kube.Get(ctx, types.NamespacedName{Name: cr.GetProviderConfigReference().Name}, pc); err != nil { - return nil, errors.Wrap(err, errGetPC) - } - - var rc *rest.Config - var err error - - switch cd := pc.Spec.Credentials; cd.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - rc, err = rest.InClusterConfig() - if err != nil { - return nil, errors.Wrap(err, errFailedToCreateRestConfig) - } - default: - kc, err := c.kcfgExtractorFn(ctx, cd.Source, c.kube, cd.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errGetCreds) - } - - if rc, err = c.newRESTConfigFn(kc); err != nil { - return nil, errors.Wrap(err, errFailedToCreateRestConfig) - } - } - - if id := pc.Spec.Identity; id != nil { - switch id.Type { - case apisv1alpha1.IdentityTypeGoogleApplicationCredentials: - switch id.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - if err := c.gcpInjectorFn(ctx, rc, nil, gke.DefaultScopes...); err != nil { - return nil, errors.Wrap(err, errFailedToInjectGoogleCredentials) - } - default: - creds, err := c.gcpExtractorFn(ctx, id.Source, c.kube, id.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errFailedToExtractGoogleCredentials) - } - - if err := c.gcpInjectorFn(ctx, rc, creds, gke.DefaultScopes...); err != nil { - return nil, errors.Wrap(err, errFailedToInjectGoogleCredentials) - } - } - case apisv1alpha1.IdentityTypeAzureServicePrincipalCredentials: - switch id.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - return nil, errors.Errorf("%s is not supported as identity source for identity type %s", - xpv1.CredentialsSourceInjectedIdentity, apisv1alpha1.IdentityTypeAzureServicePrincipalCredentials) - default: - creds, err := c.azureExtractorFn(ctx, id.Source, c.kube, id.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errFailedToExtractAzureCredentials) - } - - if err := c.azureInjectorFn(ctx, rc, creds); err != nil { - return nil, errors.Wrap(err, errFailedToInjectAzureCredentials) - } - } - default: - return nil, errors.Errorf("unknown identity type: %s", id.Type) - } - } + k, err := c.clientForProviderFn(ctx, c.kube, cr.GetProviderConfigReference().Name) - k, err := c.newKubeClientFn(rc) if err != nil { return nil, errors.Wrap(err, errNewKubernetesClient) } diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 9b1e76e2..7e151bcd 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,12 +46,7 @@ import ( ) const ( - providerName = "kubernetes-test" - providerSecretName = "kubernetes-test-secret" - providerSecretNamespace = "kubernetes-test-secret-namespace" - - providerSecretKey = "kubeconfig" - providerSecretData = "somethingsecret" + providerName = "kubernetes-test" testObjectName = "test-object" testNamespace = "test-namespace" @@ -208,11 +202,6 @@ func referenceObjectWithFinalizer(val interface{}) *unstructured.Unstructured { } func Test_connector_Connect(t *testing.T) { - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: providerSecretNamespace, Name: providerSecretName}, - Data: map[string][]byte{providerSecretKey: []byte(providerSecretData)}, - } - providerConfig := kubernetesv1alpha1.ProviderConfig{ ObjectMeta: metav1.ObjectMeta{Name: providerName}, Spec: kubernetesv1alpha1.ProviderConfigSpec{ @@ -248,16 +237,10 @@ func Test_connector_Connect(t *testing.T) { providerConfigUnknownIdentitySource.Spec.Identity.Type = "foo" type args struct { - client client.Client - kcfgExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - gcpExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - gcpInjectorFn func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error - azureExtractorFn func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) - azureInjectorFn func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error - newRESTConfigFn func(kubeconfig []byte) (*rest.Config, error) - newKubeClientFn func(config *rest.Config) (client.Client, error) - usage resource.Tracker - mg resource.Managed + client client.Client + clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) + usage resource.Tracker + mg resource.Managed } type want struct { err error @@ -283,322 +266,21 @@ func Test_connector_Connect(t *testing.T) { err: errors.Wrap(errBoom, errTrackPCUsage), }, }, - "FailedToGetProvider": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return errBoom - } - return nil - }, - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errGetPC), - }, - }, - "FailedToExtractKubeconfig": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return nil - } - if key.Name == providerSecretName && key.Namespace == providerSecretNamespace { - *obj.(*corev1.Secret) = secret - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errGetCreds), - }, - }, - "FailedToCreateRESTConfig": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return nil - } - if key.Name == providerSecretName && key.Namespace == providerSecretNamespace { - *obj.(*corev1.Secret) = secret - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToCreateRestConfig), - }, - }, - "FailedToExtractGoogleCredentials": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return nil - } - if key.Name == providerSecretName && key.Namespace == providerSecretNamespace { - *obj.(*corev1.Secret) = secret - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - gcpExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToExtractGoogleCredentials), - }, - }, - "FailedToInjectGoogleCredentials": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return nil - } - if key.Name == providerSecretName && key.Namespace == providerSecretNamespace { - *obj.(*corev1.Secret) = secret - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - gcpExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - gcpInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToInjectGoogleCredentials), - }, - }, - "FailedToInjectGoogleCredentialsWithInjectedIdentitySource": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfigGoogleInjectedIdentity - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - gcpExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - gcpInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToInjectGoogleCredentials), - }, - }, - "FailedToExtractAzureCredentials": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = *providerConfigAzure - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - azureExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToExtractAzureCredentials), - }, - }, - "FailedToInjectAzureCredentials": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = *providerConfigAzure - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - azureExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - azureInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errFailedToInjectAzureCredentials), - }, - }, - "AzureCredentialsInjectedIdentitySourceNotSupported": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfigAzureInjectedIdentity - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - azureExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - azureInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Errorf("%s is not supported as identity source for identity type %s", - xpv1.CredentialsSourceInjectedIdentity, kubernetesv1alpha1.IdentityTypeAzureServicePrincipalCredentials), - }, - }, - "FailedToInjectUnknownIdentityType": { + "Success": { args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfigUnknownIdentitySource - return nil - } - return errBoom - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return nil, nil - }, - azureExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - azureInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return errBoom + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { + return &test.MockClient{}, nil }, usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), mg: kubernetesObject(), }, want: want{ - err: errors.Errorf("unknown identity type: %s", "foo"), + err: nil, }, }, - "FailedToCreateNewKubernetesClient": { + "ErrorGettingClientForProvider": { args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == providerName { - *obj.(*kubernetesv1alpha1.ProviderConfig) = providerConfig - return nil - } - if key.Name == providerSecretName && key.Namespace == providerSecretNamespace { - *obj.(*corev1.Secret) = secret - return nil - } - return errBoom - }, - MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { - return nil - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return &rest.Config{}, nil - }, - - gcpExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - gcpInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return nil - }, - newKubeClientFn: func(config *rest.Config) (c client.Client, err error) { + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { return nil, errBoom }, usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), @@ -608,60 +290,14 @@ func Test_connector_Connect(t *testing.T) { err: errors.Wrap(errBoom, errNewKubernetesClient), }, }, - "Success": { - args: args{ - client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - switch t := obj.(type) { - case *kubernetesv1alpha1.ProviderConfig: - *t = providerConfig - case *corev1.Secret: - *t = secret - default: - return errBoom - } - return nil - }, - MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { - return nil - }, - }, - kcfgExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - newRESTConfigFn: func(kubeconfig []byte) (config *rest.Config, err error) { - return &rest.Config{}, nil - }, - gcpExtractorFn: func(ctx context.Context, src xpv1.CredentialsSource, c client.Client, ccs xpv1.CommonCredentialSelectors) ([]byte, error) { - return nil, nil - }, - gcpInjectorFn: func(ctx context.Context, rc *rest.Config, credentials []byte, scopes ...string) error { - return nil - }, - newKubeClientFn: func(config *rest.Config) (c client.Client, err error) { - return &test.MockClient{}, nil - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: nil, - }, - }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { c := &connector{ - logger: logging.NewNopLogger(), - kube: tc.args.client, - kcfgExtractorFn: tc.args.kcfgExtractorFn, - gcpExtractorFn: tc.args.gcpExtractorFn, - gcpInjectorFn: tc.args.gcpInjectorFn, - azureExtractorFn: tc.args.azureExtractorFn, - azureInjectorFn: tc.args.azureInjectorFn, - newRESTConfigFn: tc.args.newRESTConfigFn, - newKubeClientFn: tc.args.newKubeClientFn, - usage: tc.usage, + logger: logging.NewNopLogger(), + kube: tc.args.client, + clientForProviderFn: tc.args.clientForProvider, + usage: tc.usage, } _, gotErr := c.Connect(context.Background(), tc.args.mg) if diff := cmp.Diff(tc.want.err, gotErr, test.EquateErrors()); diff != "" { diff --git a/internal/controller/observedobjectcollection/reconciler.go b/internal/controller/observedobjectcollection/reconciler.go new file mode 100644 index 00000000..619e2a80 --- /dev/null +++ b/internal/controller/observedobjectcollection/reconciler.go @@ -0,0 +1,286 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observedobjectcollection + +import ( + "context" + "crypto/sha256" + "fmt" + "math/rand" + "time" + + "github.com/pkg/errors" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/controller" + xperrors "github.com/crossplane/crossplane-runtime/pkg/errors" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/crossplane/crossplane-runtime/pkg/ratelimiter" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients" +) + +const ( + errNewKubernetesClient = "cannot create new Kubernetes client" + errStatusUpdate = "cannot update status" + fieldOwner = client.FieldOwner("kubernetes.crossplane.io/observed-object-collection-controller") + membershipLabelKey = "kubernetes.crossplane.io/owned-by-collection" +) + +// Reconciler watches for ObservedObjectCollection resources +// and creates observe-only Objects for the matched items. +type Reconciler struct { + client client.Client + log logging.Logger + pollInterval func() time.Duration + clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) + observedObjectName func(collection client.Object, matchedObject client.Object) (string, error) +} + +// Setup adds a controller that reconciles ObservedObjectCollection resources. +func Setup(mgr ctrl.Manager, o controller.Options, pollJitter time.Duration) error { + name := managed.ControllerName(v1alpha1.ObservedObjectCollectionGroupKind) + + r := &Reconciler{ + client: mgr.GetClient(), + log: o.Logger, + pollInterval: func() time.Duration { + return o.PollInterval + +time.Duration((rand.Float64()-0.5)*2*float64(pollJitter)) //nolint + }, + clientForProvider: clients.ClientForProvider, + observedObjectName: observedObjectName, + } + + return ctrl.NewControllerManagedBy(mgr). + Named(name). + For(&v1alpha1.ObservedObjectCollection{}). + WithEventFilter(predicate.Or( + predicate.GenerationChangedPredicate{}, + predicate.AnnotationChangedPredicate{}, + predicate.LabelChangedPredicate{}), + ). + Complete(ratelimiter.NewReconciler(name, xperrors.WithSilentRequeueOnConflict(r), o.GlobalRateLimiter)) +} + +// Reconcile fetches objects specified by their GVK and label selector +// and creates observed-only Objects for the matches. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, error error) { //nolint:gocyclo + log := r.log.WithValues("request", req) + + defer func() { + if error == nil { + log.Info("Reconciled") + } else { + log.Info("Retry", "err", error) + } + }() + + c := &v1alpha1.ObservedObjectCollection{} + err := r.client.Get(ctx, req.NamespacedName, c) + + if err != nil { + if kerrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + if meta.WasDeleted(c) { + return ctrl.Result{}, nil + } + + if meta.IsPaused(c) { + c.Status.SetConditions(xpv1.ReconcilePaused()) + return ctrl.Result{}, errors.Wrap(r.client.Status().Update(ctx, c), errStatusUpdate) + } + + log.Info("Reconciling") + + // Get client for the referenced provider config. + clusterClient, err := r.clientForProvider(ctx, r.client, c.Spec.ProviderConfigReference.Name) + if err != nil { + werr := errors.Wrap(err, errNewKubernetesClient) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + // Fetch objects based on the set GVK and selector. + k8sobjects := &unstructured.UnstructuredList{} + k8sobjects.SetAPIVersion(c.Spec.ObserveObjects.APIVersion) + k8sobjects.SetKind(c.Spec.ObserveObjects.Kind) + selector, err := metav1.LabelSelectorAsSelector(&c.Spec.ObserveObjects.Selector) + + if err != nil { + werr := errors.Wrap(err, "error creating selector") + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + lo := client.ListOptions{LabelSelector: selector, Namespace: c.Spec.ObserveObjects.Namespace} + if err := clusterClient.List(ctx, k8sobjects, &lo); err != nil { + werr := errors.Wrapf(err, "error fetching objects for GVK %v and options %v", k8sobjects.GetObjectKind().GroupVersionKind(), lo) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + // Fetch any existing counter-part observe only Objects by collection label. + ml := map[string]string{membershipLabelKey: c.Name} + ol := &v1alpha2.ObjectList{} + if err := r.client.List(ctx, ol, client.MatchingLabels(ml)); err != nil { + werr := errors.Wrapf(err, "cannot list members matching labels %v", ml) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + // Create/update observed-only Objects for all found items. + refs := sets.New[v1alpha1.ObservedObjectReference]() + for i := range k8sobjects.Items { + o := k8sobjects.Items[i] + log.Debug("creating observed object for the matched item", "gvk", o.GroupVersionKind(), "name", o.GetName()) + name, err := r.observedObjectName(c, &o) + if err != nil { + werr := errors.Wrapf(err, "error generating name for observed object, matched object: %v", o) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + // Create patch + po, err := observedObjectPatch(name, o, c) + if err != nil { + werr := errors.Wrapf(err, "error generating patch for matched object %v", o) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + if err := r.client.Patch(ctx, po, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + werr := errors.Wrap(err, "cannot create observed object") + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + + log.Debug("created observed object", "name", po.GetName()) + refs.Insert(v1alpha1.ObservedObjectReference{Name: name}) + } + + // Remove collection members that either do not exist anymore or are no match. + for i := range ol.Items { + o := ol.Items[i] + if refs.Has(v1alpha1.ObservedObjectReference{Name: o.Name}) { + continue + } + log.Debug("Removing", "name", o.Name) + if err := r.client.Delete(ctx, &ol.Items[i]); err != nil { + werr := errors.Wrapf(err, "cannot delete observed object %v", o) + c.Status.SetConditions(xpv1.ReconcileError(werr)) + _ = r.client.Status().Update(ctx, c) + return ctrl.Result{}, werr + } + } + c.Status.SetConditions(xpv1.ReconcileSuccess(), xpv1.Available()) + + c.Status.MembershipLabel = ml + + return ctrl.Result{RequeueAfter: r.pollInterval()}, r.client.Status().Update(ctx, c) +} + +func observedObjectName(collection client.Object, matchedObject client.Object) (string, error) { + // unique object identifier + k := fmt.Sprintf("%v/%s/%s", matchedObject.GetObjectKind().GroupVersionKind(), matchedObject.GetNamespace(), matchedObject.GetName()) + // Compute sha256 hash of it and take first 56 bits. + h := sha256.New() + if _, err := h.Write([]byte(k)); err != nil { + return "", err + } + kp := fmt.Sprintf("%x", h.Sum(nil))[0:7] + // append it to the collection name + return fmt.Sprintf("%s-%s", collection.GetName(), kp), nil +} + +func observedObjectPatch(name string, matchedObject unstructured.Unstructured, collection *v1alpha1.ObservedObjectCollection) (*unstructured.Unstructured, error) { + objectManifestTemplate := `{ +"kind": "%s", +"apiVersion": "%s", +"metadata": { + "name": "%s", + "namespace": "%s" +} +}` + manifest := fmt.Sprintf(objectManifestTemplate, matchedObject.GetKind(), matchedObject.GetAPIVersion(), matchedObject.GetName(), matchedObject.GetNamespace()) + observedObject := &v1alpha2.Object{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: collection.APIVersion, + Kind: collection.Kind, + Name: collection.Name, + UID: collection.UID, + }, + }, + }, + Spec: v1alpha2.ObjectSpec{ + ResourceSpec: xpv1.ResourceSpec{ + ProviderConfigReference: &collection.Spec.ProviderConfigReference, + ManagementPolicies: []xpv1.ManagementAction{xpv1.ManagementActionObserve}, + }, + ForProvider: v1alpha2.ObjectParameters{ + Manifest: runtime.RawExtension{ + Raw: []byte(manifest), + }, + }, + }, + } + labels := map[string]string{ + membershipLabelKey: collection.Name, + } + if t := collection.Spec.Template; t != nil { + for k, v := range t.Metadata.Labels { + labels[k] = v + } + if len(t.Metadata.Annotations) > 0 { + observedObject.SetAnnotations(t.Metadata.Annotations) + } + } + observedObject.SetLabels(labels) + v, err := runtime.DefaultUnstructuredConverter.ToUnstructured(observedObject) + if err != nil { + return nil, errors.Wrap(err, "cannot convert to unstructured") + } + u := &unstructured.Unstructured{Object: v} + u.SetGroupVersionKind(v1alpha2.ObjectGroupVersionKind) + u.SetName(observedObject.Name) + return u, nil +} diff --git a/internal/controller/observedobjectcollection/reconciler_test.go b/internal/controller/observedobjectcollection/reconciler_test.go new file mode 100644 index 00000000..f7faaa30 --- /dev/null +++ b/internal/controller/observedobjectcollection/reconciler_test.go @@ -0,0 +1,402 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observedobjectcollection + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/test" + + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" +) + +func TestReconciler(t *testing.T) { + collectionName := types.NamespacedName{Name: "col"} + errBoom := fmt.Errorf("error reading") + pollIterval := 10 * time.Second + objectAPIVersion := "v1" + objectKind := "Foo" + type args struct { + client *test.MockClient + } + type want struct { + r reconcile.Result + err error + } + cases := map[string]struct { + reason string + args args + want want + }{ + "ErrorGetCollection": { + reason: "We should return error.", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + return errBoom + }, + }, + }, + want: want{ + err: errBoom, + }, + }, + "CollectionNotFound": { + reason: "We should not return an error if the collection resource was not found.", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + return kerrors.NewNotFound(schema.GroupResource{}, "") + }, + }, + }, + }, + "CreateObservedObjects": { + reason: "Create observed-only object from the matched objects.", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key != collectionName { + return fmt.Errorf("Expected %v, but got %v", collectionName, key) + } + c := obj.(*v1alpha1.ObservedObjectCollection) + c.Spec = v1alpha1.ObservedObjectCollectionSpec{ + ObserveObjects: v1alpha1.ObserveObjectCriteria{ + APIVersion: objectAPIVersion, + Kind: objectKind, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + }, + } + c.Name = collectionName.Name + return nil + }, + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if olist, ok := list.(*v1alpha2.ObjectList); ok { + olist.Items = append(olist.Items, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo0"}}, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo1"}}) + return nil + } + ulist := list.(*unstructured.UnstructuredList) + if ulist.GetAPIVersion() != "v1" || ulist.GetKind() != "Foo" { + return fmt.Errorf("Unexpected GVK %v", ulist.GroupVersionKind()) + } + for i := 0; i < 2; i++ { + item := unstructured.Unstructured{} + item.SetKind(ulist.GetKind()) + item.SetAPIVersion(ulist.GetAPIVersion()) + item.SetName(fmt.Sprintf("foo%d", i)) + item.SetUID(types.UID(uuid.New().String())) + ulist.Items = append(ulist.Items, item) + } + return nil + }, + MockPatch: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if patch != client.Apply { + return fmt.Errorf("Expected SSA patch, but got: %v", patch) + } + u := obj.(*unstructured.Unstructured) + if u.GroupVersionKind() != v1alpha2.ObjectGroupVersionKind { + return fmt.Errorf("Expected gvk %v, but got %v", v1alpha2.ObjectGroupVersionKind, u.GroupVersionKind()) + } + if l := u.GetLabels()[membershipLabelKey]; l != collectionName.Name { + return fmt.Errorf("Expecting membership label %v but got %v", collectionName.Name, l) + } + if s := sets.New[string]("col-foo0", "col-foo1"); !s.Has(u.GetName()) { + return fmt.Errorf("Expecting one of %v, but got %v", s.UnsortedList(), u.GetName()) + } + manifest, found, err := unstructured.NestedMap(u.Object, "spec", "forProvider", "manifest") + if err != nil { + return err + } + if !found { + return fmt.Errorf("Manifest not found") + } + if apiVersion := manifest["apiVersion"]; apiVersion != objectAPIVersion { + return fmt.Errorf("Manifest apiVersion should be %v, but got: %v", objectAPIVersion, apiVersion) + } + if kind := manifest["kind"]; kind != objectKind { + return fmt.Errorf("Manifest kind should be %v, but got: %v", objectKind, kind) + } + manifestMetadataName, _, _ := unstructured.NestedString(manifest, "metadata", "name") + if s := sets.New[string]("foo0", "foo1"); !s.Has(manifestMetadataName) { + return fmt.Errorf("Expecting one of %v, but got %v", s.UnsortedList(), manifestMetadataName) + } + return nil + }, + MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + c := obj.(*v1alpha1.ObservedObjectCollection) + if cnd := c.Status.GetCondition(xpv1.TypeSynced); cnd.Status != corev1.ConditionTrue { + panic(fmt.Sprintf("Object sync condition not true: %v", cnd.Message)) + } + if cnd := c.Status.GetCondition(xpv1.TypeReady); cnd.Status != corev1.ConditionTrue { + panic(fmt.Sprintf("Object ready condition not true: %v", cnd.Message)) + } + if v := c.Status.MembershipLabel[membershipLabelKey]; v != collectionName.Name { + panic(fmt.Sprintf("Expected membership label %v but got %v", collectionName.Name, v)) + } + return nil + }, + }, + }, + want: want{ + r: reconcile.Result{RequeueAfter: pollIterval}, + }, + }, + "RemoveNotMatchedObservedObjects": { + reason: "Remove observe-only objects that either not exist or are not matched anymore", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key != collectionName { + return fmt.Errorf("Expected %v, but got %v", collectionName, key) + } + c := obj.(*v1alpha1.ObservedObjectCollection) + c.Spec = v1alpha1.ObservedObjectCollectionSpec{ + ObserveObjects: v1alpha1.ObserveObjectCriteria{ + APIVersion: objectAPIVersion, + Kind: objectKind, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + }, + } + c.Name = collectionName.Name + return nil + }, + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if olist, ok := list.(*v1alpha2.ObjectList); ok { + olist.Items = append(olist.Items, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo0"}}, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo1"}}, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo2"}}) + return nil + } + ulist := list.(*unstructured.UnstructuredList) + if ulist.GetAPIVersion() != "v1" || ulist.GetKind() != "Foo" { + return fmt.Errorf("Unexpected GVK %v", ulist.GroupVersionKind()) + } + for i := 0; i < 2; i++ { + item := unstructured.Unstructured{} + item.SetKind(ulist.GetKind()) + item.SetAPIVersion(ulist.GetAPIVersion()) + item.SetName(fmt.Sprintf("foo%d", i)) + item.SetUID(types.UID(uuid.New().String())) + ulist.Items = append(ulist.Items, item) + } + return nil + }, + MockDelete: func(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if obj.GetName() != "col-foo2" { + return fmt.Errorf("Expected to remove col-foo2, but got %v", obj.GetName()) + } + return nil + }, + MockPatch: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if patch != client.Apply { + return fmt.Errorf("Expected SSA patch, but got: %v", patch) + } + u := obj.(*unstructured.Unstructured) + if u.GroupVersionKind() != v1alpha2.ObjectGroupVersionKind { + return fmt.Errorf("Expected gvk %v, but got %v", v1alpha2.ObjectGroupVersionKind, u.GroupVersionKind()) + } + if s := sets.New[string]("col-foo0", "col-foo1"); !s.Has(u.GetName()) { + return fmt.Errorf("Expecting one of %v, but got %v", s.UnsortedList(), u.GetName()) + } + manifest, found, err := unstructured.NestedMap(u.Object, "spec", "forProvider", "manifest") + if err != nil { + return err + } + if !found { + return fmt.Errorf("Manifest not found") + } + if apiVersion := manifest["apiVersion"]; apiVersion != objectAPIVersion { + return fmt.Errorf("Manifest apiVersion should be %v, but got: %v", objectAPIVersion, apiVersion) + } + if kind := manifest["kind"]; kind != objectKind { + return fmt.Errorf("Manifest kind should be %v, but got: %v", objectKind, kind) + } + manifestMetadataName, _, _ := unstructured.NestedString(manifest, "metadata", "name") + if s := sets.New[string]("foo0", "foo1"); !s.Has(manifestMetadataName) { + return fmt.Errorf("Expecting one of %v, but got %v", s.UnsortedList(), manifestMetadataName) + } + return nil + }, + MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + c := obj.(*v1alpha1.ObservedObjectCollection) + if cnd := c.Status.GetCondition(xpv1.TypeSynced); cnd.Status != corev1.ConditionTrue { + return fmt.Errorf("Object sync condition not true: %v", cnd.Message) + } + if cnd := c.Status.GetCondition(xpv1.TypeReady); cnd.Status != corev1.ConditionTrue { + return fmt.Errorf("Object ready condition not true: %v", cnd.Message) + } + if v := c.Status.MembershipLabel[membershipLabelKey]; v != collectionName.Name { + return fmt.Errorf("Expected membership label %v but got %v", collectionName.Name, v) + } + return nil + }, + }, + }, + want: want{ + r: reconcile.Result{RequeueAfter: pollIterval}, + }, + }, + "ErrorListingObjects": { + reason: "Return error and set collection status if listing objects produces error.", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key != collectionName { + return fmt.Errorf("Expected %v, but got %v", collectionName, key) + } + c := obj.(*v1alpha1.ObservedObjectCollection) + c.Spec = v1alpha1.ObservedObjectCollectionSpec{ + ObserveObjects: v1alpha1.ObserveObjectCriteria{ + APIVersion: objectAPIVersion, + Kind: objectKind, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + }, + } + c.Name = collectionName.Name + return nil + }, + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return errBoom + }, + MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + c := obj.(*v1alpha1.ObservedObjectCollection) + if cnd := c.Status.GetCondition(xpv1.TypeSynced); cnd.Status != corev1.ConditionFalse { + panic(fmt.Sprintf("Object sync condition not true: %v", cnd.Message)) + } + + return nil + }, + }, + }, + want: want{ + err: errBoom, + }, + }, + "ErrorCreatingObservedObjects": { + reason: "Return error and update collection status if error occurs while creating observe only object", + args: args{ + client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key != collectionName { + return fmt.Errorf("Expected %v, but got %v", collectionName, key) + } + c := obj.(*v1alpha1.ObservedObjectCollection) + c.Spec = v1alpha1.ObservedObjectCollectionSpec{ + ObserveObjects: v1alpha1.ObserveObjectCriteria{ + APIVersion: objectAPIVersion, + Kind: objectKind, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + }, + }, + } + c.Name = collectionName.Name + return nil + }, + MockList: func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if olist, ok := list.(*v1alpha2.ObjectList); ok { + olist.Items = append(olist.Items, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo0"}}, v1alpha2.Object{ObjectMeta: metav1.ObjectMeta{Name: "col-foo1"}}) + return nil + } + ulist := list.(*unstructured.UnstructuredList) + if ulist.GetAPIVersion() != "v1" || ulist.GetKind() != "Foo" { + return fmt.Errorf("Unexpected GVK %v", ulist.GroupVersionKind()) + } + for i := 0; i < 2; i++ { + item := unstructured.Unstructured{} + item.SetKind(ulist.GetKind()) + item.SetAPIVersion(ulist.GetAPIVersion()) + item.SetName(fmt.Sprintf("foo%d", i)) + item.SetUID(types.UID(uuid.New().String())) + ulist.Items = append(ulist.Items, item) + } + return nil + }, + MockPatch: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return errBoom + }, + MockStatusUpdate: func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + c := obj.(*v1alpha1.ObservedObjectCollection) + if cnd := c.Status.GetCondition(xpv1.TypeSynced); cnd.Status != corev1.ConditionFalse { + panic(fmt.Sprintf("Object sync condition not true: %v", cnd.Message)) + } + return nil + }, + }, + }, + want: want{ + err: errBoom, + }, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + r := &Reconciler{ + client: tc.args.client, + log: logging.NewNopLogger(), + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { + return tc.args.client, nil + }, + observedObjectName: func(collection client.Object, matchedObject client.Object) (string, error) { + return fmt.Sprintf("%s-%s", collection.GetName(), matchedObject.GetName()), nil + }, + pollInterval: func() time.Duration { + return pollIterval + }, + } + got, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: collectionName}) + if !errors.Is(err, tc.want.err) { + t.Errorf("\n%s\nr.Reconcile(...): want error: %v, got error: %v", tc.reason, tc.want.err, err) + } + if diff := cmp.Diff(tc.want.r, got, test.EquateErrors()); diff != "" { + t.Errorf("\n%s\nr.Reconcile(...): -want, +got:\n%s", tc.reason, diff) + } + }) + } +} diff --git a/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml b/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml new file mode 100644 index 00000000..59384dbe --- /dev/null +++ b/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml @@ -0,0 +1,257 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: observedobjectcollections.kubernetes.crossplane.io +spec: + group: kubernetes.crossplane.io + names: + categories: + - crossplane + - managed + - kubernetes + kind: ObservedObjectCollection + listKind: ObservedObjectCollectionList + plural: observedobjectcollections + singular: observedobjectcollection + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.kind + name: KIND + type: string + - jsonPath: .spec.apiVersion + name: APIVERSION + priority: 1 + type: string + - jsonPath: .spec.providerConfigRef.name + name: PROVIDERCONFIG + type: string + - jsonPath: .status.conditions[?(@.type=='Synced')].status + name: SYNCED + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: READY + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: A ObservedObjectCollection is a provider Kubernetes API type + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ObservedObjectCollectionSpec defines the desired state of + ObservedObjectCollection + properties: + objectTemplate: + description: Template when defined is used for creating Object instances + properties: + metadata: + description: Objects metadata + properties: + annotations: + additionalProperties: + type: string + description: Annotations of an object + type: object + labels: + additionalProperties: + type: string + description: Labels of an object + type: object + type: object + type: object + observeObjects: + description: |- + ObserveObjects declares what criteria object need to fulfil + to become a member of this collection + properties: + apiVersion: + description: APIVersion of objects that should be matched by the + selector + minLength: 1 + type: string + kind: + description: Kind of objects that should be matched by the selector + minLength: 1 + type: string + namespace: + description: |- + Namespace where to look for objects. + If omitted, search is performed across all namespaces. + For cluster-scoped objects, omit it. + type: string + selector: + description: Selector defines the criteria for including objects + into the collection + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - apiVersion + - kind + - selector + type: object + providerConfigRef: + default: + name: default + description: |- + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. + properties: + name: + description: Name of the referenced object. + type: string + policy: + description: Policies for referencing. + properties: + resolution: + default: Required + description: |- + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. + enum: + - Required + - Optional + type: string + resolve: + description: |- + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. + enum: + - Always + - IfNotPresent + type: string + type: object + required: + - name + type: object + required: + - observeObjects + type: object + status: + description: ObservedObjectCollectionStatus represents the observed state + of a ObservedObjectCollection + properties: + conditions: + description: Conditions of the resource. + items: + description: A Condition that may apply to a resource. + properties: + lastTransitionTime: + description: |- + LastTransitionTime is the last time this condition transitioned from one + status to another. + format: date-time + type: string + message: + description: |- + A Message containing details about this condition's last transition from + one status to another, if any. + type: string + reason: + description: A Reason for this condition's last transition from + one status to another. + type: string + status: + description: Status of this condition; is it currently True, + False, or Unknown? + type: string + type: + description: |- + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + membershipLabel: + additionalProperties: + type: string + description: |- + MembershipLabel is the label set on each member of this collection + and can be used for fetching them. + type: object + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: metadata.name max length is 63 + rule: size(self.metadata.name) < 64 + served: true + storage: true + subresources: + status: {} From 4cbf04037685a9db7d6ec7a9eff297e22eb76b84 Mon Sep 17 00:00:00 2001 From: Jack Jackson Date: Tue, 7 May 2024 22:48:33 -0700 Subject: [PATCH 14/36] Update README with correct installation instructions See https://github.com/crossplane-contrib/provider-kubernetes/issues/233 Signed-off-by: Jack Jackson Signed-off-by: Alexander Brandstedt --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 94f6d2a4..99ea8803 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ so using the Crossplane CLI in a Kubernetes cluster where Crossplane is installed: ```console -kubectl crossplane install provider crossplanecontrib/provider-kubernetes:main +kubectl crossplane xpkg install provider xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.13.0 ``` You may also manually install `provider-kubernetes` by creating a `Provider` directly: @@ -25,7 +25,7 @@ kind: Provider metadata: name: provider-kubernetes spec: - package: "crossplanecontrib/provider-kubernetes:main" + package: xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.13.0 ``` ## Developing locally From 91d84384887023b9194cb97bb7f7b626d37e46a3 Mon Sep 17 00:00:00 2001 From: Jack Jackson Date: Wed, 8 May 2024 13:51:27 +0100 Subject: [PATCH 15/36] Update README.md Co-authored-by: Jared Watts Signed-off-by: Jack Jackson Signed-off-by: Alexander Brandstedt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99ea8803..332c6e23 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ so using the Crossplane CLI in a Kubernetes cluster where Crossplane is installed: ```console -kubectl crossplane xpkg install provider xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.13.0 +crossplane xpkg install provider xpkg.upbound.io/crossplane-contrib/provider-kubernetes:v0.13.0 ``` You may also manually install `provider-kubernetes` by creating a `Provider` directly: From ba12c53ba37698e523f9a6a59e815128e3ee0eff Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Wed, 1 May 2024 00:32:46 +0300 Subject: [PATCH 16/36] Bump build submodule and dependencies to latest Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- Makefile | 4 ++-- build | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5534faec..4bc23a4f 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,8 @@ GOLANGCILINT_VERSION = 1.55.2 # ==================================================================================== # Setup Kubernetes tools -KIND_VERSION = v0.18.0 -UP_VERSION = v0.21.0 +KIND_VERSION = v0.22.0 +UP_VERSION = v0.28.0 UPTEST_VERSION = v0.9.0 UP_CHANNEL = stable USE_HELM3 = true diff --git a/build b/build index 7233e364..cdee0068 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit 7233e36491dc8298d33f1feb1bf8c5056a960cac +Subproject commit cdee00685cfc90fce40d3f6a5eb9c8f044edbaa3 From d1539b6cc2c4bbe7ce042fe39d4dcaf3c8d3093f Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Wed, 1 May 2024 00:33:06 +0300 Subject: [PATCH 17/36] Implement watching referenced resources Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- go.mod | 2 +- internal/controller/object/indexes.go | 105 ++++++++++++ internal/controller/object/informers.go | 205 ++++++++++++++++++++++++ internal/controller/object/object.go | 67 +++++++- 4 files changed, 377 insertions(+), 2 deletions(-) create mode 100644 internal/controller/object/indexes.go create mode 100644 internal/controller/object/informers.go diff --git a/go.mod b/go.mod index aef6678d..3d5e4b54 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( golang.org/x/oauth2 v0.16.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 k8s.io/api v0.29.1 + k8s.io/apiextensions-apiserver v0.29.1 k8s.io/apimachinery v0.29.1 k8s.io/client-go v0.29.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b @@ -95,7 +96,6 @@ require ( gopkg.in/retry.v1 v1.0.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.1 // indirect k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go new file mode 100644 index 00000000..f831c5dc --- /dev/null +++ b/internal/controller/object/indexes.go @@ -0,0 +1,105 @@ +/* +Copyright 2023 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package object + +import ( + "context" + "fmt" + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/crossplane/crossplane-runtime/pkg/logging" +) + +const ( + // objectRefGVKsIndex is an index of all GroupKinds that + // are in use by a Composition. It indexes from spec.resourceRefs, not + // from spec.resources. Hence, it will also work with functions. + objectRefGVKsIndex = "objectsRefsGVKs" + // objectsRefsIndex is an index of resourceRefs that are owned + // by a composite. + objectsRefsIndex = "objectsRefs" +) + +var ( + _ client.IndexerFunc = IndexReferencedResourceRefGVKs + _ client.IndexerFunc = IndexReferencesResourcesRefs +) + +// IndexReferencedResourceRefGVKs assumes the passed object is a composite. It +// returns gvk keys for every resource referenced in the composite. +func IndexReferencedResourceRefGVKs(o client.Object) []string { + obj, ok := o.(*v1alpha2.Object) + if !ok { + return nil // should never happen + } + refs := obj.Spec.References + keys := make([]string, 0, len(refs)) + for _, ref := range refs { + refAPIVersion, refKind, _, _ := getReferenceInfo(ref) + group, version := parseAPIVersion(refAPIVersion) + keys = append(keys, schema.GroupVersionKind{Group: group, Version: version, Kind: refKind}.String()) + } + // unification is done by the informer. + return keys +} + +// IndexReferencesResourcesRefs assumes the passed object is a composite. It +// returns keys for every composed resource referenced in the composite. +func IndexReferencesResourcesRefs(o client.Object) []string { + obj, ok := o.(*v1alpha2.Object) + if !ok { + return nil // should never happen + } + refs := obj.Spec.References + keys := make([]string, 0, len(refs)) + for _, ref := range refs { + refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) + keys = append(keys, refKey(refNamespace, refName, refKind, refAPIVersion)) + } + return keys +} + +func refKey(ns, name, kind, apiVersion string) string { + return fmt.Sprintf("%s.%s.%s.%s", name, ns, kind, apiVersion) +} + +func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { + return func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { + rGVK := ev.ObjectNew.GetObjectKind().GroupVersionKind() + key := refKey(ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) + + objects := v1alpha2.ObjectList{} + if err := ca.List(ctx, &objects, client.MatchingFields{objectsRefsIndex: key}); err != nil { + log.Debug("cannot list objects related to a reference change", "error", err, "fieldSelector", objectsRefsIndex+"="+key) + return + } + + // queue those Objects for reconciliation + for _, o := range objects.Items { + log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}}) + } + } +} diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go new file mode 100644 index 00000000..0426ac0e --- /dev/null +++ b/internal/controller/object/informers.go @@ -0,0 +1,205 @@ +package object + +import ( + "context" + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + "strings" + "sync" + + "github.com/google/uuid" + kunstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + kcache "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + cache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/crossplane/crossplane-runtime/pkg/controller" + "github.com/crossplane/crossplane-runtime/pkg/logging" +) + +// referencedResourceInformers manages composed resource informers referenced by +// composite resources. It serves as an event source for realtime notifications +// of changed composed resources, with the composite reconcilers as sinks. +// It keeps composed resource informers alive as long as there are composites +// referencing them. In parallel, the composite reconcilers keep track of +// references to composed resources, and inform referencedResourceInformers about +// them via the WatchReferencedResources method. +type referencedResourceInformers struct { + log logging.Logger + cluster cluster.Cluster + + gvkRoutedCache *controller.GVKRoutedCache + + lock sync.RWMutex // everything below is protected by this lock + + // cdCaches holds the composed resource informers. These are dynamically + // started and stopped based on the composites that reference them. + cdCaches map[schema.GroupVersionKind]cdCache + objectsCache cache.Cache + sinks map[string]func(ev runtimeevent.UpdateEvent) // by some uid +} + +type cdCache struct { + cache cache.Cache + cancelFn context.CancelFunc +} + +var _ source.Source = &referencedResourceInformers{} + +// Start implements source.Source, i.e. starting referencedResourceInformers as +// source with h as the sink of update events. It keeps sending events until +// ctx is done. +// Note that Start can be called multiple times to deliver events to multiple +// (composite resource) controllers. +func (i *referencedResourceInformers) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, ps ...predicate.Predicate) error { + id := uuid.New().String() + + i.lock.Lock() + defer i.lock.Unlock() + i.sinks[id] = func(ev runtimeevent.UpdateEvent) { + for _, p := range ps { + if !p.Update(ev) { + return + } + } + h.Update(ctx, ev, q) + } + + go func() { + <-ctx.Done() + + i.lock.Lock() + defer i.lock.Unlock() + delete(i.sinks, id) + }() + + return nil +} + +// WatchReferencedResources starts informers for the given composed resource GVKs. +// The is wired into the composite reconciler, which will call this method on +// every reconcile to make referencedResourceInformers aware of the composed +// resources the given composite resource references. +// +// Note that this complements cleanupReferencedResourceInformers which regularly +// garbage collects composed resource informers that are no longer referenced by +// any composite. +func (i *referencedResourceInformers) WatchReferencedResources(gvks ...schema.GroupVersionKind) { + i.lock.RLock() + defer i.lock.RUnlock() + + // start new informers + for _, gvk := range gvks { + if _, found := i.cdCaches[gvk]; found { + continue + } + + log := i.log.WithValues("gvk", gvk.String()) + + ca, err := cache.New(i.cluster.GetConfig(), cache.Options{}) + if err != nil { + log.Debug("failed creating a cache", "error", err) + continue + } + + // don't forget to call cancelFn in error cases to avoid leaks. In the + // happy case it's called from the go routine starting the cache below. + ctx, cancelFn := context.WithCancel(context.Background()) + + u := kunstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + inf, err := ca.GetInformer(ctx, &u, cache.BlockUntilSynced(false)) // don't block. We wait in the go routine below. + if err != nil { + cancelFn() + log.Debug("failed getting informer", "error", err) + continue + } + + if _, err := inf.AddEventHandler(kcache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj interface{}) { + old := oldObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. + obj := newObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. + if old.GetResourceVersion() == obj.GetResourceVersion() { + return + } + + i.lock.RLock() + defer i.lock.RUnlock() + + ev := runtimeevent.UpdateEvent{ + ObjectOld: old, + ObjectNew: obj, + } + for _, handleFn := range i.sinks { + handleFn(ev) + } + }, + }); err != nil { + cancelFn() + log.Debug("failed adding event handler", "error", err) + continue + } + + go func() { + defer cancelFn() + + log.Info("Starting composed resource watch") + _ = ca.Start(ctx) + }() + + i.cdCaches[gvk] = cdCache{ + cache: ca, + cancelFn: cancelFn, + } + + // wait for in the background, and only when synced add to the routed cache + go func(gvk schema.GroupVersionKind) { + if synced := ca.WaitForCacheSync(ctx); synced { + log.Debug("Composed resource cache synced") + i.gvkRoutedCache.AddDelegate(gvk, ca) + } + }(gvk) + } +} + +// cleanupReferencedResourceInformers garbage collects composed resource informers +// that are no longer referenced by any composite resource. +// +// Note that this complements WatchReferencedResources which starts informers for +// the composed resources referenced by a composite resource. +func (i *referencedResourceInformers) cleanupReferencedResourceInformers(ctx context.Context) { + // stop old informers + for gvk, inf := range i.cdCaches { + list := v1alpha2.ObjectList{} + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: gvk.String()}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+gvk.String()) + } + + if len(list.Items) > 0 { + continue + } + + inf.cancelFn() + i.gvkRoutedCache.RemoveDelegate(gvk) + i.log.Info("Stopped referenced resource watch", "gvk", gvk.String()) + delete(i.cdCaches, gvk) + } +} + +func parseAPIVersion(v string) (string, string) { + parts := strings.SplitN(v, "/", 2) + switch len(parts) { + case 1: + return "", parts[0] + case 2: + return parts[0], parts[1] + default: + return "", "" + } +} diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index b1fd63f6..733f3717 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -20,7 +20,13 @@ import ( "context" "encoding/base64" "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" "math/rand" + runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "strings" "time" @@ -82,12 +88,47 @@ const ( errSanitizeSecretData = "cannot sanitize secret data" ) +// KindObserver tracks kinds of referenced composed resources in order to start +// watches for them for realtime events. +type KindObserver interface { + // WatchReferencedResources starts a watch of the given kinds to trigger + // reconciles when a referenced object of those kinds changes. + WatchReferencedResources(kind ...schema.GroupVersionKind) +} + // Setup adds a controller that reconciles Object managed resources. func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJitter time.Duration) error { name := managed.ControllerName(v1alpha2.ObjectGroupKind) + l := o.Logger.WithValues("controller", name) cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} + ca := mgr.GetCache() + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { + return errors.Wrap(err, "cannot add index for object reference GVKs") + } + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectsRefsIndex, IndexReferencesResourcesRefs); err != nil { + return errors.Wrap(err, "cannot add index for object references") + } + + i := referencedResourceInformers{ + log: l, + cluster: mgr, + + gvkRoutedCache: controller.NewGVKRoutedCache(mgr.GetScheme(), mgr.GetCache()), + cdCaches: make(map[schema.GroupVersionKind]cdCache), + objectsCache: ca, + sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), + } + + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + // Run every 5 minutes. + wait.UntilWithContext(ctx, i.cleanupReferencedResourceInformers, 5*time.Second) + return nil + })); err != nil { + return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") + } + reconcilerOptions := []managed.ReconcilerOption{ managed.WithExternalConnecter(&connector{ logger: o.Logger, @@ -95,6 +136,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit kube: mgr.GetClient(), usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), clientForProviderFn: clients.ClientForProvider, + kindObserver: &i, }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), @@ -107,7 +149,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit // https://github.com/crossplane/crossplane-runtime/blob/7fcb8c5cad6fc4abb6649813b92ab92e1832d368/pkg/reconciler/managed/reconciler.go#L573 return pollInterval + time.Duration((rand.Float64()-0.5)*2*float64(pollJitter)) //nolint G404 // No need for secure randomness }), - managed.WithLogger(o.Logger.WithValues("controller", name)), + managed.WithLogger(l), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithConnectionPublishers(cps...), } @@ -125,6 +167,11 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit Named(name). WithOptions(o.ForControllerRuntime()). For(&v1alpha2.Object{}). + WatchesRawSource(&i, handler.Funcs{ + UpdateFunc: func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { + enqueueObjectsForReferences(ca, l)(ctx, ev, q) + }, + }). Complete(ratelimiter.NewReconciler(name, r, o.GlobalRateLimiter)) } @@ -134,6 +181,8 @@ type connector struct { logger logging.Logger sanitizeSecrets bool + kindObserver KindObserver + clientForProviderFn func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) } @@ -164,6 +213,8 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E }, localClient: c.kube, sanitizeSecrets: c.sanitizeSecrets, + + kindObserver: c.kindObserver, }, nil } @@ -173,6 +224,8 @@ type external struct { // localClient is specifically used to connect to local cluster localClient client.Client sanitizeSecrets bool + + kindObserver KindObserver } func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.ExternalObservation, error) { @@ -407,6 +460,7 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) c.logger.Debug("Resolving referencies.") // Loop through references to resolve each referenced resource + var gvks []schema.GroupVersionKind for _, ref := range obj.Spec.References { if ref.DependsOn == nil && ref.PatchesFrom == nil { continue @@ -432,6 +486,17 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) return errors.Wrap(err, errPatchFromReferencedResource) } } + + g, v := parseAPIVersion(refAPIVersion) + gvks = append(gvks, schema.GroupVersionKind{ + Group: g, + Version: v, + Kind: refKind, + }) + } + + if c.kindObserver != nil { + c.kindObserver.WatchReferencedResources(gvks...) } return nil From ac7ee5db17ba2563a6179910e14e7296769edc92 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 11:49:33 +0300 Subject: [PATCH 18/36] Watch managed resources as well Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- go.mod | 2 +- internal/clients/client.go | 35 ++++++++++- internal/controller/object/indexes.go | 46 ++++++++++++--- internal/controller/object/informers.go | 59 +++++++++++-------- internal/controller/object/object.go | 51 +++++++++------- .../observedobjectcollection/reconciler.go | 2 +- 6 files changed, 138 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index 3d5e4b54..aef6678d 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( golang.org/x/oauth2 v0.16.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 k8s.io/api v0.29.1 - k8s.io/apiextensions-apiserver v0.29.1 k8s.io/apimachinery v0.29.1 k8s.io/client-go v0.29.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b @@ -96,6 +95,7 @@ require ( gopkg.in/retry.v1 v1.0.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.29.1 // indirect k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect diff --git a/internal/clients/client.go b/internal/clients/client.go index eff1a1a0..0c410a17 100644 --- a/internal/clients/client.go +++ b/internal/clients/client.go @@ -107,8 +107,28 @@ func restConfigFromAPIConfig(c *api.Config) (*rest.Config, error) { return config, nil } +type Cluster interface { + // GetConfig returns an initialized Config + GetConfig() *rest.Config +} + +type ClusterClient interface { + client.Client + resource.Applicator + Cluster +} + +type clusterClient struct { + resource.ClientApplicator + config *rest.Config +} + +func (c *clusterClient) GetConfig() *rest.Config { + return c.config +} + // ClientForProvider returns the client for the given provider config -func ClientForProvider(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { //nolint:gocyclo +func ClientForProvider(ctx context.Context, inclusterClient client.Client, providerConfigName string) (ClusterClient, error) { //nolint:gocyclo pc := &v1alpha1.ProviderConfig{} if err := inclusterClient.Get(ctx, types.NamespacedName{Name: providerConfigName}, pc); err != nil { return nil, errors.Wrap(err, errGetPC) @@ -171,6 +191,15 @@ func ClientForProvider(ctx context.Context, inclusterClient client.Client, provi return nil, errors.Errorf("unknown identity type: %s", id.Type) } } - - return NewKubeClient(rc) + k, err := NewKubeClient(rc) + if err != nil { + return nil, errors.Wrap(err, "cannot create Kubernetes client") + } + return &clusterClient{ + ClientApplicator: resource.ClientApplicator{ + Client: k, + Applicator: resource.NewAPIPatchingApplicator(k), + }, + config: rc, + }, nil } diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index f831c5dc..01c80007 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -59,12 +58,25 @@ func IndexReferencedResourceRefGVKs(o client.Object) []string { for _, ref := range refs { refAPIVersion, refKind, _, _ := getReferenceInfo(ref) group, version := parseAPIVersion(refAPIVersion) - keys = append(keys, schema.GroupVersionKind{Group: group, Version: version, Kind: refKind}.String()) + // References are always on control plane, so we don't pass the config name. + keys = append(keys, refKeyGKV("", refKind, group, version)) } + + d, err := getDesired(obj) + if err == nil { + keys = append(keys, refKeyGKV(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer. + } else { + // TODO: what to do here? + } + // unification is done by the informer. return keys } +func refKeyGKV(config, kind, group, version string) string { + return fmt.Sprintf("%s.%s.%s.%s", config, kind, group, version) +} + // IndexReferencesResourcesRefs assumes the passed object is a composite. It // returns keys for every composed resource referenced in the composite. func IndexReferencesResourcesRefs(o client.Object) []string { @@ -76,30 +88,50 @@ func IndexReferencesResourcesRefs(o client.Object) []string { keys := make([]string, 0, len(refs)) for _, ref := range refs { refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) - keys = append(keys, refKey(refNamespace, refName, refKind, refAPIVersion)) + // References are always on control plane, so we don't pass the config name. + keys = append(keys, refKey("", refNamespace, refName, refKind, refAPIVersion)) + } + d, err := getDesired(obj) + if err == nil { + keys = append(keys, refKey(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer. + } else { + // TODO: what to do here? } return keys } -func refKey(ns, name, kind, apiVersion string) string { - return fmt.Sprintf("%s.%s.%s.%s", name, ns, kind, apiVersion) +func refKey(config, ns, name, kind, apiVersion string) string { + return fmt.Sprintf("%s.%s.%s.%s.%s", config, name, ns, kind, apiVersion) } func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { return func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { + config := getConfigName(ev.ObjectNew) + // "config" is the provider config name, which is the value for the + // provider config ref annotation. It will be empty for objects that + // are not controlled by the "Object" resource, i.e. referenced objects. rGVK := ev.ObjectNew.GetObjectKind().GroupVersionKind() - key := refKey(ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) + key := refKey(config, ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) objects := v1alpha2.ObjectList{} if err := ca.List(ctx, &objects, client.MatchingFields{objectsRefsIndex: key}); err != nil { log.Debug("cannot list objects related to a reference change", "error", err, "fieldSelector", objectsRefsIndex+"="+key) return } + log.Info("Will enqueue objects for references", "len", len(objects.Items)) // queue those Objects for reconciliation for _, o := range objects.Items { - log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) + log.Info("Enqueueing Object because referenced resource changed", "len", len(objects.Items), "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}}) } } } + +func getConfigName(o client.Object) string { + ann := o.GetAnnotations() + if ann == nil { + return "" + } + return ann["kubernetes.crossplane.io/provider-config-ref"] +} diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 0426ac0e..31305633 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -2,7 +2,6 @@ package object import ( "context" - "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "strings" "sync" @@ -11,15 +10,17 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" kcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - cache "sigs.k8s.io/controller-runtime/pkg/cache" + + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/cluster" + runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/crossplane/crossplane-runtime/pkg/controller" + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients" "github.com/crossplane/crossplane-runtime/pkg/logging" ) @@ -32,19 +33,26 @@ import ( // them via the WatchReferencedResources method. type referencedResourceInformers struct { log logging.Logger - cluster cluster.Cluster - - gvkRoutedCache *controller.GVKRoutedCache + cluster clients.Cluster lock sync.RWMutex // everything below is protected by this lock // cdCaches holds the composed resource informers. These are dynamically // started and stopped based on the composites that reference them. - cdCaches map[schema.GroupVersionKind]cdCache + cdCaches map[gvkWithConfig]cdCache objectsCache cache.Cache sinks map[string]func(ev runtimeevent.UpdateEvent) // by some uid } +type gvkWithConfig struct { + config string + gvk schema.GroupVersionKind +} + +func (g gvkWithConfig) String() string { + return g.config + "." + g.gvk.String() +} + type cdCache struct { cache cache.Cache cancelFn context.CancelFunc @@ -90,19 +98,24 @@ func (i *referencedResourceInformers) Start(ctx context.Context, h handler.Event // Note that this complements cleanupReferencedResourceInformers which regularly // garbage collects composed resource informers that are no longer referenced by // any composite. -func (i *referencedResourceInformers) WatchReferencedResources(gvks ...schema.GroupVersionKind) { +func (i *referencedResourceInformers) WatchReferencedResources(cluster clients.Cluster, gcs ...gvkWithConfig) { i.lock.RLock() defer i.lock.RUnlock() // start new informers - for _, gvk := range gvks { - if _, found := i.cdCaches[gvk]; found { + for _, gc := range gcs { + if _, found := i.cdCaches[gc]; found { continue } - log := i.log.WithValues("gvk", gvk.String()) + log := i.log.WithValues("config", gc.config, "gvk", gc.gvk.String()) + + if cluster == nil { + // Default to control plane cluster. + cluster = i.cluster + } - ca, err := cache.New(i.cluster.GetConfig(), cache.Options{}) + ca, err := cache.New(cluster.GetConfig(), cache.Options{}) if err != nil { log.Debug("failed creating a cache", "error", err) continue @@ -113,7 +126,7 @@ func (i *referencedResourceInformers) WatchReferencedResources(gvks ...schema.Gr ctx, cancelFn := context.WithCancel(context.Background()) u := kunstructured.Unstructured{} - u.SetGroupVersionKind(gvk) + u.SetGroupVersionKind(gc.gvk) inf, err := ca.GetInformer(ctx, &u, cache.BlockUntilSynced(false)) // don't block. We wait in the go routine below. if err != nil { cancelFn() @@ -153,18 +166,17 @@ func (i *referencedResourceInformers) WatchReferencedResources(gvks ...schema.Gr _ = ca.Start(ctx) }() - i.cdCaches[gvk] = cdCache{ + i.cdCaches[gc] = cdCache{ cache: ca, cancelFn: cancelFn, } // wait for in the background, and only when synced add to the routed cache - go func(gvk schema.GroupVersionKind) { + go func() { if synced := ca.WaitForCacheSync(ctx); synced { log.Debug("Composed resource cache synced") - i.gvkRoutedCache.AddDelegate(gvk, ca) } - }(gvk) + }() } } @@ -175,10 +187,10 @@ func (i *referencedResourceInformers) WatchReferencedResources(gvks ...schema.Gr // the composed resources referenced by a composite resource. func (i *referencedResourceInformers) cleanupReferencedResourceInformers(ctx context.Context) { // stop old informers - for gvk, inf := range i.cdCaches { + for gc, inf := range i.cdCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: gvk.String()}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+gvk.String()) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: refKeyGKV(gc.config, gc.gvk.Kind, gc.gvk.Group, gc.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+gc.String()) } if len(list.Items) > 0 { @@ -186,9 +198,8 @@ func (i *referencedResourceInformers) cleanupReferencedResourceInformers(ctx con } inf.cancelFn() - i.gvkRoutedCache.RemoveDelegate(gvk) - i.log.Info("Stopped referenced resource watch", "gvk", gvk.String()) - delete(i.cdCaches, gvk) + i.log.Info("Stopped referenced resource watch", "gc", gc.String()) + delete(i.cdCaches, gc) } } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 733f3717..b183ba84 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -93,7 +93,7 @@ const ( type KindObserver interface { // WatchReferencedResources starts a watch of the given kinds to trigger // reconciles when a referenced object of those kinds changes. - WatchReferencedResources(kind ...schema.GroupVersionKind) + WatchReferencedResources(cluster clients.Cluster, gcs ...gvkWithConfig) } // Setup adds a controller that reconciles Object managed resources. @@ -115,10 +115,9 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit log: l, cluster: mgr, - gvkRoutedCache: controller.NewGVKRoutedCache(mgr.GetScheme(), mgr.GetCache()), - cdCaches: make(map[schema.GroupVersionKind]cdCache), - objectsCache: ca, - sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), + cdCaches: make(map[gvkWithConfig]cdCache), + objectsCache: ca, + sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), } if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { @@ -183,7 +182,7 @@ type connector struct { kindObserver KindObserver - clientForProviderFn func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) + clientForProviderFn func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) } func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.ExternalClient, error) { //nolint:gocyclo @@ -206,11 +205,8 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E } return &external{ - logger: c.logger, - client: resource.ClientApplicator{ - Client: k, - Applicator: resource.NewAPIPatchingApplicator(k), - }, + logger: c.logger, + client: k, localClient: c.kube, sanitizeSecrets: c.sanitizeSecrets, @@ -220,7 +216,7 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E type external struct { logger logging.Logger - client resource.ClientApplicator + client clients.ClusterClient // localClient is specifically used to connect to local cluster localClient client.Client sanitizeSecrets bool @@ -235,7 +231,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex } c.logger.Debug("Observing", "resource", cr) - + if err := c.resolveReferencies(ctx, cr); err != nil { return managed.ExternalObservation{}, errors.Wrap(err, errResolveResourceReferences) } @@ -256,6 +252,13 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex return managed.ExternalObservation{ResourceExists: false}, nil } + if c.kindObserver != nil { + c.kindObserver.WatchReferencedResources(c.client, gvkWithConfig{ + config: cr.Spec.ProviderConfigReference.Name, + gvk: observed.GroupVersionKind(), + }) + } + if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, errGetObject) } @@ -285,7 +288,8 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext } meta.AddAnnotations(obj, map[string]string{ - v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), + v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), + "kubernetes.crossplane.io/provider-config-ref": cr.Spec.ProviderConfigReference.Name, }) if err := c.client.Create(ctx, obj); err != nil { @@ -309,7 +313,8 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext } meta.AddAnnotations(obj, map[string]string{ - v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), + v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), + "kubernetes.crossplane.io/provider-config-ref": cr.Spec.ProviderConfigReference.Name, }) if err := c.client.Apply(ctx, obj); err != nil { @@ -344,6 +349,7 @@ func getDesired(obj *v1alpha2.Object) (*unstructured.Unstructured, error) { if desired.GetName() == "" { desired.SetName(obj.Name) } + return desired, nil } @@ -460,7 +466,7 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) c.logger.Debug("Resolving referencies.") // Loop through references to resolve each referenced resource - var gvks []schema.GroupVersionKind + var gcs []gvkWithConfig for _, ref := range obj.Spec.References { if ref.DependsOn == nil && ref.PatchesFrom == nil { continue @@ -488,15 +494,18 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) } g, v := parseAPIVersion(refAPIVersion) - gvks = append(gvks, schema.GroupVersionKind{ - Group: g, - Version: v, - Kind: refKind, + gcs = append(gcs, gvkWithConfig{ + config: "", + gvk: schema.GroupVersionKind{ + Group: g, + Version: v, + Kind: refKind, + }, }) } if c.kindObserver != nil { - c.kindObserver.WatchReferencedResources(gvks...) + c.kindObserver.WatchReferencedResources(nil, gcs...) } return nil diff --git a/internal/controller/observedobjectcollection/reconciler.go b/internal/controller/observedobjectcollection/reconciler.go index 619e2a80..7620051c 100644 --- a/internal/controller/observedobjectcollection/reconciler.go +++ b/internal/controller/observedobjectcollection/reconciler.go @@ -59,7 +59,7 @@ type Reconciler struct { client client.Client log logging.Logger pollInterval func() time.Duration - clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) + clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) observedObjectName func(collection client.Object, matchedObject client.Object) (string, error) } From 8fb68a4980aeace5d2227ed659ba588b3b0e105c Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 13:00:51 +0300 Subject: [PATCH 19/36] Do not start multiple caches against the same host Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/indexes.go | 2 - internal/controller/object/informers.go | 71 +++++++++++++------------ internal/controller/object/object.go | 48 ++++++++--------- 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index 01c80007..b0be1711 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -118,8 +118,6 @@ func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx co log.Debug("cannot list objects related to a reference change", "error", err, "fieldSelector", objectsRefsIndex+"="+key) return } - log.Info("Will enqueue objects for references", "len", len(objects.Items)) - // queue those Objects for reconciliation for _, o := range objects.Items { log.Info("Enqueueing Object because referenced resource changed", "len", len(objects.Items), "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 31305633..c48b1742 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -2,6 +2,7 @@ package object import ( "context" + "k8s.io/client-go/rest" "strings" "sync" @@ -20,7 +21,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients" "github.com/crossplane/crossplane-runtime/pkg/logging" ) @@ -32,30 +32,32 @@ import ( // references to composed resources, and inform referencedResourceInformers about // them via the WatchReferencedResources method. type referencedResourceInformers struct { - log logging.Logger - cluster clients.Cluster + log logging.Logger + config *rest.Config lock sync.RWMutex // everything below is protected by this lock - // cdCaches holds the composed resource informers. These are dynamically - // started and stopped based on the composites that reference them. - cdCaches map[gvkWithConfig]cdCache - objectsCache cache.Cache - sinks map[string]func(ev runtimeevent.UpdateEvent) // by some uid + // resourceCaches holds the managed resource informers. These are + // dynamically started and stopped based on the Objects that reference OR + // manages them. + resourceCaches map[gvkWithHost]resourceCache + objectsCache cache.Cache + sinks map[string]func(ev runtimeevent.UpdateEvent) // by some uid } -type gvkWithConfig struct { - config string - gvk schema.GroupVersionKind +type gvkWithHost struct { + host string + gvk schema.GroupVersionKind } -func (g gvkWithConfig) String() string { - return g.config + "." + g.gvk.String() -} - -type cdCache struct { +type resourceCache struct { cache cache.Cache cancelFn context.CancelFunc + + // Which provider config was used to create this cache. We will use this + // information to figure out whether there are Objects relying on this cache + // left during garbage collection of caches. + providerConfig string } var _ source.Source = &referencedResourceInformers{} @@ -98,24 +100,23 @@ func (i *referencedResourceInformers) Start(ctx context.Context, h handler.Event // Note that this complements cleanupReferencedResourceInformers which regularly // garbage collects composed resource informers that are no longer referenced by // any composite. -func (i *referencedResourceInformers) WatchReferencedResources(cluster clients.Cluster, gcs ...gvkWithConfig) { +func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { i.lock.RLock() defer i.lock.RUnlock() + if rc == nil { + rc = i.config + } + // start new informers - for _, gc := range gcs { - if _, found := i.cdCaches[gc]; found { + for _, gvk := range gvks { + if _, found := i.resourceCaches[gvkWithHost{host: rc.Host, gvk: gvk}]; found { continue } - log := i.log.WithValues("config", gc.config, "gvk", gc.gvk.String()) - - if cluster == nil { - // Default to control plane cluster. - cluster = i.cluster - } + log := i.log.WithValues("host", rc.Host, "gvk", gvk.String()) - ca, err := cache.New(cluster.GetConfig(), cache.Options{}) + ca, err := cache.New(rc, cache.Options{}) if err != nil { log.Debug("failed creating a cache", "error", err) continue @@ -126,7 +127,7 @@ func (i *referencedResourceInformers) WatchReferencedResources(cluster clients.C ctx, cancelFn := context.WithCancel(context.Background()) u := kunstructured.Unstructured{} - u.SetGroupVersionKind(gc.gvk) + u.SetGroupVersionKind(gvk) inf, err := ca.GetInformer(ctx, &u, cache.BlockUntilSynced(false)) // don't block. We wait in the go routine below. if err != nil { cancelFn() @@ -166,9 +167,11 @@ func (i *referencedResourceInformers) WatchReferencedResources(cluster clients.C _ = ca.Start(ctx) }() - i.cdCaches[gc] = cdCache{ + i.resourceCaches[gvkWithHost{host: rc.Host, gvk: gvk}] = resourceCache{ cache: ca, cancelFn: cancelFn, + + providerConfig: providerConfig, } // wait for in the background, and only when synced add to the routed cache @@ -187,19 +190,19 @@ func (i *referencedResourceInformers) WatchReferencedResources(cluster clients.C // the composed resources referenced by a composite resource. func (i *referencedResourceInformers) cleanupReferencedResourceInformers(ctx context.Context) { // stop old informers - for gc, inf := range i.cdCaches { + for gh, ca := range i.resourceCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: refKeyGKV(gc.config, gc.gvk.Kind, gc.gvk.Group, gc.gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+gc.String()) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) } if len(list.Items) > 0 { continue } - inf.cancelFn() - i.log.Info("Stopped referenced resource watch", "gc", gc.String()) - delete(i.cdCaches, gc) + ca.cancelFn() + i.log.Info("Stopped referenced resource watch", "provider config", ca.providerConfig, "host", gh.host, "gvk", gh.gvk) + delete(i.resourceCaches, gh) } } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index b183ba84..b72f1956 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -22,6 +22,7 @@ import ( "fmt" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" "math/rand" runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" @@ -91,9 +92,9 @@ const ( // KindObserver tracks kinds of referenced composed resources in order to start // watches for them for realtime events. type KindObserver interface { - // WatchReferencedResources starts a watch of the given kinds to trigger - // reconciles when a referenced object of those kinds changes. - WatchReferencedResources(cluster clients.Cluster, gcs ...gvkWithConfig) + // WatchResources starts a watch of the given kinds to trigger reconciles + // when a referenced or managed objects of those kinds changes. + WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) } // Setup adds a controller that reconciles Object managed resources. @@ -112,12 +113,12 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit } i := referencedResourceInformers{ - log: l, - cluster: mgr, + log: l, + config: mgr.GetConfig(), - cdCaches: make(map[gvkWithConfig]cdCache), - objectsCache: ca, - sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), + objectsCache: ca, + resourceCaches: make(map[gvkWithHost]resourceCache), + sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), } if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { @@ -252,17 +253,16 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex return managed.ExternalObservation{ResourceExists: false}, nil } - if c.kindObserver != nil { - c.kindObserver.WatchReferencedResources(c.client, gvkWithConfig{ - config: cr.Spec.ProviderConfigReference.Name, - gvk: observed.GroupVersionKind(), - }) - } - if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, errGetObject) } + // We know the resource exists, so we can start watching it for realtime + // events. + if c.kindObserver != nil { + c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, observed.GroupVersionKind()) + } + if err = c.setObserved(cr, observed); err != nil { return managed.ExternalObservation{}, err } @@ -466,7 +466,7 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) c.logger.Debug("Resolving referencies.") // Loop through references to resolve each referenced resource - var gcs []gvkWithConfig + var gvks []schema.GroupVersionKind for _, ref := range obj.Spec.References { if ref.DependsOn == nil && ref.PatchesFrom == nil { continue @@ -494,18 +494,18 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) } g, v := parseAPIVersion(refAPIVersion) - gcs = append(gcs, gvkWithConfig{ - config: "", - gvk: schema.GroupVersionKind{ - Group: g, - Version: v, - Kind: refKind, - }, + gvks = append(gvks, schema.GroupVersionKind{ + Group: g, + Version: v, + Kind: refKind, }) } if c.kindObserver != nil { - c.kindObserver.WatchReferencedResources(nil, gcs...) + // Referenced resources always live on the control plane (i.e. local cluster), + // so we don't pass an extra rest config or provider config with the + // watch call. + c.kindObserver.WatchResources(nil, "", gvks...) } return nil From a3fa10e0c30c9b942d53bbd2a7e78f548f9e6287 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 13:58:20 +0300 Subject: [PATCH 20/36] Do not attempt resolving references if object is being deleted Fixes #138 Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/object.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index b72f1956..8f8ecffe 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -232,9 +232,12 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex } c.logger.Debug("Observing", "resource", cr) - - if err := c.resolveReferencies(ctx, cr); err != nil { - return managed.ExternalObservation{}, errors.Wrap(err, errResolveResourceReferences) + + if !meta.WasDeleted(cr) { + // If the object is not being deleted, we need to resolve references + if err := c.resolveReferencies(ctx, cr); err != nil { + return managed.ExternalObservation{}, errors.Wrap(err, errResolveResourceReferences) + } } desired, err := getDesired(cr) From c1c94252edd95ceb565cec8eb8ccf8ee193b0650 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 14:20:46 +0300 Subject: [PATCH 21/36] Put resource watching behind feature flag Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 24 ++++--- internal/controller/object/informers.go | 7 +- internal/controller/object/object.go | 96 +++++++++++++------------ internal/features/features.go | 23 ++++++ 4 files changed, 94 insertions(+), 56 deletions(-) create mode 100644 internal/features/features.go diff --git a/cmd/provider/main.go b/cmd/provider/main.go index ab235cc7..742b3db0 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "github.com/crossplane-contrib/provider-kubernetes/internal/features" "io" "os" "path/filepath" @@ -50,15 +51,17 @@ const ( func main() { var ( - app = kingpin.New(filepath.Base(os.Args[0]), "Template support for Crossplane.").DefaultEnvars() - debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() - syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() - pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration() - pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() - leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() - maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("100").Int() + app = kingpin.New(filepath.Base(os.Args[0]), "Template support for Crossplane.").DefaultEnvars() + debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() + syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() + pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration() + pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() + leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() + maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("100").Int() + sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() + enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() - sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() + enableWatches = app.Flag("enable-watches", "Enable support for watching resources.").Default("false").Envar("ENABLE_WATCHES").Bool() ) kingpin.MustParse(app.Parse(os.Args[1:])) @@ -135,6 +138,11 @@ func main() { log.Info("Beta feature enabled", "flag", feature.EnableBetaManagementPolicies) } + if *enableWatches { + o.Features.Enable(features.EnableAlphaWatches) + log.Info("Alpha feature enabled", "flag", features.EnableAlphaWatches) + } + // NOTE(lsviben): We are registering the conversion webhook with v1alpha1 // Object. As far as I can see and based on some tests, it doesn't matter // which version we use here. Leaving it as v1alpha1 as it will be easy to diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index c48b1742..8fe57d94 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -97,7 +97,7 @@ func (i *referencedResourceInformers) Start(ctx context.Context, h handler.Event // every reconcile to make referencedResourceInformers aware of the composed // resources the given composite resource references. // -// Note that this complements cleanupReferencedResourceInformers which regularly +// Note that this complements cleanupResourceInformers which regularly // garbage collects composed resource informers that are no longer referenced by // any composite. func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { @@ -183,13 +183,14 @@ func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerCo } } -// cleanupReferencedResourceInformers garbage collects composed resource informers +// cleanupResourceInformers garbage collects composed resource informers // that are no longer referenced by any composite resource. // // Note that this complements WatchReferencedResources which starts informers for // the composed resources referenced by a composite resource. -func (i *referencedResourceInformers) cleanupReferencedResourceInformers(ctx context.Context) { +func (i *referencedResourceInformers) cleanupResourceInformers(ctx context.Context) { // stop old informers + i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) for gh, ca := range i.resourceCaches { list := v1alpha2.ObjectList{} if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 8f8ecffe..e2acd2b8 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "github.com/crossplane-contrib/provider-kubernetes/internal/features" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" @@ -104,40 +105,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit cps := []managed.ConnectionPublisher{managed.NewAPISecretPublisher(mgr.GetClient(), mgr.GetScheme())} - ca := mgr.GetCache() - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { - return errors.Wrap(err, "cannot add index for object reference GVKs") - } - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectsRefsIndex, IndexReferencesResourcesRefs); err != nil { - return errors.Wrap(err, "cannot add index for object references") - } - - i := referencedResourceInformers{ - log: l, - config: mgr.GetConfig(), - - objectsCache: ca, - resourceCaches: make(map[gvkWithHost]resourceCache), - sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), - } - - if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - // Run every 5 minutes. - wait.UntilWithContext(ctx, i.cleanupReferencedResourceInformers, 5*time.Second) - return nil - })); err != nil { - return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") - } - reconcilerOptions := []managed.ReconcilerOption{ - managed.WithExternalConnecter(&connector{ - logger: o.Logger, - sanitizeSecrets: sanitizeSecrets, - kube: mgr.GetClient(), - usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), - clientForProviderFn: clients.ClientForProvider, - kindObserver: &i, - }), managed.WithFinalizer(&objFinalizer{client: mgr.GetClient()}), managed.WithPollInterval(o.PollInterval), managed.WithPollIntervalHook(func(mg resource.Managed, pollInterval time.Duration) time.Duration { @@ -154,25 +122,63 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit managed.WithConnectionPublishers(cps...), } - if o.Features.Enabled(feature.EnableBetaManagementPolicies) { - reconcilerOptions = append(reconcilerOptions, managed.WithManagementPolicies()) + conn := &connector{ + logger: o.Logger, + sanitizeSecrets: sanitizeSecrets, + kube: mgr.GetClient(), + usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), + clientForProviderFn: clients.ClientForProvider, } - r := managed.NewReconciler(mgr, - resource.ManagedKind(v1alpha2.ObjectGroupVersionKind), - reconcilerOptions..., - ) - - return ctrl.NewControllerManagedBy(mgr). + cb := ctrl.NewControllerManagedBy(mgr). Named(name). WithOptions(o.ForControllerRuntime()). - For(&v1alpha2.Object{}). - WatchesRawSource(&i, handler.Funcs{ + For(&v1alpha2.Object{}) + + if o.Features.Enabled(features.EnableAlphaWatches) { + ca := mgr.GetCache() + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { + return errors.Wrap(err, "cannot add index for object reference GVKs") + } + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectsRefsIndex, IndexReferencesResourcesRefs); err != nil { + return errors.Wrap(err, "cannot add index for object references") + } + + i := referencedResourceInformers{ + log: l, + config: mgr.GetConfig(), + + objectsCache: ca, + resourceCaches: make(map[gvkWithHost]resourceCache), + sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), + } + conn.kindObserver = &i + + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + // Run every 5 minutes. + // TODO: Fix the period after development is done. + wait.UntilWithContext(ctx, i.cleanupResourceInformers, 5*time.Second) + return nil + })); err != nil { + return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") + } + + cb = cb.WatchesRawSource(&i, handler.Funcs{ UpdateFunc: func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { enqueueObjectsForReferences(ca, l)(ctx, ev, q) }, - }). - Complete(ratelimiter.NewReconciler(name, r, o.GlobalRateLimiter)) + }) + } + reconcilerOptions = append(reconcilerOptions, managed.WithExternalConnecter(conn)) + + if o.Features.Enabled(feature.EnableBetaManagementPolicies) { + reconcilerOptions = append(reconcilerOptions, managed.WithManagementPolicies()) + } + + return cb.Complete(ratelimiter.NewReconciler(name, managed.NewReconciler(mgr, + resource.ManagedKind(v1alpha2.ObjectGroupVersionKind), + reconcilerOptions..., + ), o.GlobalRateLimiter)) } type connector struct { diff --git a/internal/features/features.go b/internal/features/features.go new file mode 100644 index 00000000..3c294983 --- /dev/null +++ b/internal/features/features.go @@ -0,0 +1,23 @@ +/* + Copyright 2022 The Crossplane Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package features + +import "github.com/crossplane/crossplane-runtime/pkg/feature" + +// Feature flags. +const ( + // EnableAlphaWatches enables alpha support for watching referenced and + // managed resources. + EnableAlphaWatches feature.Flag = "EnableAlphaWatches" +) From a604c64fd9461263e36732d5d76a30b9de1c53be Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 15:24:58 +0300 Subject: [PATCH 22/36] Fix unit tests and code comments Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 2 +- internal/clients/{client.go => clients.go} | 15 +- internal/controller/object/indexes.go | 74 ++-- internal/controller/object/informers.go | 88 ++-- internal/controller/object/object.go | 58 ++- internal/controller/object/object_test.go | 381 ++++++++++-------- .../reconciler_test.go | 10 +- 7 files changed, 352 insertions(+), 276 deletions(-) rename internal/clients/{client.go => clients.go} (92%) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index 742b3db0..fca9670b 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -17,7 +17,6 @@ limitations under the License. package main import ( - "github.com/crossplane-contrib/provider-kubernetes/internal/features" "io" "os" "path/filepath" @@ -39,6 +38,7 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha1" object "github.com/crossplane-contrib/provider-kubernetes/internal/controller" + "github.com/crossplane-contrib/provider-kubernetes/internal/features" _ "k8s.io/client-go/plugin/pkg/client/auth" ) diff --git a/internal/clients/client.go b/internal/clients/clients.go similarity index 92% rename from internal/clients/client.go rename to internal/clients/clients.go index 0c410a17..34b22d30 100644 --- a/internal/clients/client.go +++ b/internal/clients/clients.go @@ -107,23 +107,28 @@ func restConfigFromAPIConfig(c *api.Config) (*rest.Config, error) { return config, nil } -type Cluster interface { +// RestConfigGetter is an interface that provides a REST config. +type RestConfigGetter interface { // GetConfig returns an initialized Config GetConfig() *rest.Config } +// ClusterClient is a client that can be used to interact with a Kubernetes +// cluster including getting its rest config. type ClusterClient interface { client.Client resource.Applicator - Cluster + RestConfigGetter } -type clusterClient struct { +// ApplicatorClientWithConfig is a ClusterClient that also has a rest config. +type ApplicatorClientWithConfig struct { resource.ClientApplicator config *rest.Config } -func (c *clusterClient) GetConfig() *rest.Config { +// GetConfig returns the rest config for the client. +func (c *ApplicatorClientWithConfig) GetConfig() *rest.Config { return c.config } @@ -195,7 +200,7 @@ func ClientForProvider(ctx context.Context, inclusterClient client.Client, provi if err != nil { return nil, errors.Wrap(err, "cannot create Kubernetes client") } - return &clusterClient{ + return &ApplicatorClientWithConfig{ ClientApplicator: resource.ClientApplicator{ Client: k, Applicator: resource.NewAPIPatchingApplicator(k), diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index b0be1711..d127f81c 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -19,7 +19,6 @@ package object import ( "context" "fmt" - "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" @@ -29,16 +28,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/crossplane/crossplane-runtime/pkg/logging" + + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" ) const ( - // objectRefGVKsIndex is an index of all GroupKinds that - // are in use by a Composition. It indexes from spec.resourceRefs, not - // from spec.resources. Hence, it will also work with functions. - objectRefGVKsIndex = "objectsRefsGVKs" - // objectsRefsIndex is an index of resourceRefs that are owned - // by a composite. - objectsRefsIndex = "objectsRefs" + // resourceRefGVKsIndex is an index of all GroupKinds that + // are in use by an Object. + resourceRefGVKsIndex = "objectsRefsGVKs" + // resourceRefsIndex is an index of resourceRefs that are referenced or + // managed by an Object. + resourceRefsIndex = "objectsRefs" ) var ( @@ -46,13 +46,15 @@ var ( _ client.IndexerFunc = IndexReferencesResourcesRefs ) -// IndexReferencedResourceRefGVKs assumes the passed object is a composite. It -// returns gvk keys for every resource referenced in the composite. +// IndexReferencedResourceRefGVKs assumes the passed object is an Object. It +// returns gvk keys for every resource referenced or managed by the Object. func IndexReferencedResourceRefGVKs(o client.Object) []string { obj, ok := o.(*v1alpha2.Object) if !ok { return nil // should never happen } + + // Index references. refs := obj.Spec.References keys := make([]string, 0, len(refs)) for _, ref := range refs { @@ -62,46 +64,48 @@ func IndexReferencedResourceRefGVKs(o client.Object) []string { keys = append(keys, refKeyGKV("", refKind, group, version)) } - d, err := getDesired(obj) - if err == nil { - keys = append(keys, refKeyGKV(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer. - } else { - // TODO: what to do here? - } + // Index the desired object. + // We don't expect errors here, as the getDesired function is already called + // in the reconciler and the desired object already validated. + d, _ := getDesired(obj) + keys = append(keys, refKeyGKV(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer. // unification is done by the informer. return keys } -func refKeyGKV(config, kind, group, version string) string { - return fmt.Sprintf("%s.%s.%s.%s", config, kind, group, version) +func refKeyGKV(providerConfig, kind, group, version string) string { + return fmt.Sprintf("%s.%s.%s.%s", providerConfig, kind, group, version) } -// IndexReferencesResourcesRefs assumes the passed object is a composite. It -// returns keys for every composed resource referenced in the composite. +// IndexReferencesResourcesRefs assumes the passed object is an Object. It +// returns keys for every resource referenced or managed by the Object. func IndexReferencesResourcesRefs(o client.Object) []string { obj, ok := o.(*v1alpha2.Object) if !ok { return nil // should never happen } + + // Index references. refs := obj.Spec.References keys := make([]string, 0, len(refs)) for _, ref := range refs { refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) - // References are always on control plane, so we don't pass the config name. + // References are always on control plane, so we don't pass the provider config name. keys = append(keys, refKey("", refNamespace, refName, refKind, refAPIVersion)) } - d, err := getDesired(obj) - if err == nil { - keys = append(keys, refKey(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer. - } else { - // TODO: what to do here? - } + + // Index the desired object. + // We don't expect errors here, as the getDesired function is already called + // in the reconciler and the desired object already validated. + d, _ := getDesired(obj) + keys = append(keys, refKey(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer. + return keys } -func refKey(config, ns, name, kind, apiVersion string) string { - return fmt.Sprintf("%s.%s.%s.%s.%s", config, name, ns, kind, apiVersion) +func refKey(providerConfig, ns, name, kind, apiVersion string) string { + return fmt.Sprintf("%s.%s.%s.%s.%s", providerConfig, name, ns, kind, apiVersion) } func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { @@ -114,22 +118,22 @@ func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx co key := refKey(config, ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) objects := v1alpha2.ObjectList{} - if err := ca.List(ctx, &objects, client.MatchingFields{objectsRefsIndex: key}); err != nil { - log.Debug("cannot list objects related to a reference change", "error", err, "fieldSelector", objectsRefsIndex+"="+key) + if err := ca.List(ctx, &objects, client.MatchingFields{resourceRefsIndex: key}); err != nil { + log.Debug("cannot list objects related to a reference change", "error", err, "fieldSelector", resourceRefsIndex+"="+key) return } // queue those Objects for reconciliation for _, o := range objects.Items { - log.Info("Enqueueing Object because referenced resource changed", "len", len(objects.Items), "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) + log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}}) } } } func getConfigName(o client.Object) string { - ann := o.GetAnnotations() - if ann == nil { + a := o.GetAnnotations() + if a == nil { return "" } - return ann["kubernetes.crossplane.io/provider-config-ref"] + return a[AnnotationKeyProviderConfigRef] } diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 8fe57d94..bcca7fde 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -2,47 +2,45 @@ package object import ( "context" - "k8s.io/client-go/rest" + "github.com/crossplane/crossplane-runtime/pkg/errors" "strings" "sync" - "github.com/google/uuid" kunstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" kcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "github.com/crossplane/crossplane-runtime/pkg/logging" + + "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" ) -// referencedResourceInformers manages composed resource informers referenced by -// composite resources. It serves as an event source for realtime notifications -// of changed composed resources, with the composite reconcilers as sinks. -// It keeps composed resource informers alive as long as there are composites -// referencing them. In parallel, the composite reconcilers keep track of -// references to composed resources, and inform referencedResourceInformers about -// them via the WatchReferencedResources method. -type referencedResourceInformers struct { +// resourceInformers manages resource informers referenced or managed +// by Objects. It serves as an event source for realtime notifications of +// changed resources, with the Object reconcilers as sinks. +// It keeps resource informers alive as long as there are Objects referencing +// them. In parallel, the Object reconcilers keep track of references to +// resources, and inform resourceInformers about them via the +// WatchReferencedResources method. +type resourceInformers struct { log logging.Logger config *rest.Config lock sync.RWMutex // everything below is protected by this lock - // resourceCaches holds the managed resource informers. These are - // dynamically started and stopped based on the Objects that reference OR - // manages them. + // resourceCaches holds the resource caches. These are dynamically started + // and stopped based on the Objects that reference or managing them. resourceCaches map[gvkWithHost]resourceCache objectsCache cache.Cache - sinks map[string]func(ev runtimeevent.UpdateEvent) // by some uid + sink func(ev runtimeevent.UpdateEvent) } type gvkWithHost struct { @@ -60,19 +58,18 @@ type resourceCache struct { providerConfig string } -var _ source.Source = &referencedResourceInformers{} +var _ source.Source = &resourceInformers{} -// Start implements source.Source, i.e. starting referencedResourceInformers as +// Start implements source.Source, i.e. starting resourceInformers as // source with h as the sink of update events. It keeps sending events until // ctx is done. -// Note that Start can be called multiple times to deliver events to multiple -// (composite resource) controllers. -func (i *referencedResourceInformers) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, ps ...predicate.Predicate) error { - id := uuid.New().String() - +func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, ps ...predicate.Predicate) error { i.lock.Lock() defer i.lock.Unlock() - i.sinks[id] = func(ev runtimeevent.UpdateEvent) { + if i.sink != nil { + return errors.New("source already started, cannot start it again") + } + i.sink = func(ev runtimeevent.UpdateEvent) { for _, p := range ps { if !p.Update(ev) { return @@ -86,21 +83,22 @@ func (i *referencedResourceInformers) Start(ctx context.Context, h handler.Event i.lock.Lock() defer i.lock.Unlock() - delete(i.sinks, id) + i.sink = nil }() return nil } -// WatchReferencedResources starts informers for the given composed resource GVKs. -// The is wired into the composite reconciler, which will call this method on -// every reconcile to make referencedResourceInformers aware of the composed -// resources the given composite resource references. +// WatchResources starts informers for the given resource GVKs for the given +// cluster (i.e. rest.Config & providerConfig). +// The is wired into the Object reconciler, which will call this method on +// every reconcile to make resourceInformers aware of the referenced or managed +// resources of the given Object. // // Note that this complements cleanupResourceInformers which regularly -// garbage collects composed resource informers that are no longer referenced by -// any composite. -func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { +// garbage collects resource informers that are no longer referenced by +// any Object. +func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { i.lock.RLock() defer i.lock.RUnlock() @@ -150,9 +148,7 @@ func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerCo ObjectOld: old, ObjectNew: obj, } - for _, handleFn := range i.sinks { - handleFn(ev) - } + i.sink(ev) }, }); err != nil { cancelFn() @@ -163,7 +159,7 @@ func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerCo go func() { defer cancelFn() - log.Info("Starting composed resource watch") + log.Info("Starting resource watch") _ = ca.Start(ctx) }() @@ -177,24 +173,24 @@ func (i *referencedResourceInformers) WatchResources(rc *rest.Config, providerCo // wait for in the background, and only when synced add to the routed cache go func() { if synced := ca.WaitForCacheSync(ctx); synced { - log.Debug("Composed resource cache synced") + log.Debug("Resource cache synced") } }() } } -// cleanupResourceInformers garbage collects composed resource informers -// that are no longer referenced by any composite resource. +// cleanupResourceInformers garbage collects resource informers that are no +// longer referenced by any Object. // -// Note that this complements WatchReferencedResources which starts informers for -// the composed resources referenced by a composite resource. -func (i *referencedResourceInformers) cleanupResourceInformers(ctx context.Context) { +// Note that this complements WatchResources which starts informers for +// the resources referenced or managed by an Object. +func (i *resourceInformers) cleanupResourceInformers(ctx context.Context) { // stop old informers i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) for gh, ca := range i.resourceCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{objectRefGVKsIndex: refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", objectRefGVKsIndex+"="+refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) } if len(list.Items) > 0 { @@ -202,7 +198,7 @@ func (i *referencedResourceInformers) cleanupResourceInformers(ctx context.Conte } ca.cancelFn() - i.log.Info("Stopped referenced resource watch", "provider config", ca.providerConfig, "host", gh.host, "gvk", gh.gvk) + i.log.Info("Stopped resource watch", "provider config", ca.providerConfig, "host", gh.host, "gvk", gh.gvk) delete(i.resourceCaches, gh) } } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index e2acd2b8..690ddf9a 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -20,15 +20,7 @@ import ( "context" "encoding/base64" "fmt" - "github.com/crossplane-contrib/provider-kubernetes/internal/features" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/rest" - "k8s.io/client-go/util/workqueue" "math/rand" - runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" "strings" "time" @@ -37,11 +29,18 @@ import ( "k8s.io/apimachinery/pkg/api/equality" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/controller" @@ -57,6 +56,15 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" apisv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" "github.com/crossplane-contrib/provider-kubernetes/internal/clients" + "github.com/crossplane-contrib/provider-kubernetes/internal/features" +) + +const ( + // AnnotationKeyProviderConfigRef is the annotation key for the provider config + // reference on the managed kubernetes resource. We use this to trigger the + // reconciler for the Object with that provider config in case of watch + // events. Only used when alpha watches are enabled. + AnnotationKeyProviderConfigRef = "kubernetes.crossplane.io/provider-config-ref" ) const ( @@ -137,27 +145,25 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit if o.Features.Enabled(features.EnableAlphaWatches) { ca := mgr.GetCache() - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { return errors.Wrap(err, "cannot add index for object reference GVKs") } - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, objectsRefsIndex, IndexReferencesResourcesRefs); err != nil { + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefsIndex, IndexReferencesResourcesRefs); err != nil { return errors.Wrap(err, "cannot add index for object references") } - i := referencedResourceInformers{ + i := resourceInformers{ log: l, config: mgr.GetConfig(), objectsCache: ca, resourceCaches: make(map[gvkWithHost]resourceCache), - sinks: make(map[string]func(ev runtimeevent.UpdateEvent)), } conn.kindObserver = &i if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { // Run every 5 minutes. - // TODO: Fix the period after development is done. - wait.UntilWithContext(ctx, i.cleanupResourceInformers, 5*time.Second) + wait.UntilWithContext(ctx, i.cleanupResourceInformers, 5*time.Minute) return nil })); err != nil { return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") @@ -267,7 +273,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex } // We know the resource exists, so we can start watching it for realtime - // events. + // events if we have the kindObserver (i.e. watches enabled). if c.kindObserver != nil { c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, observed.GroupVersionKind()) } @@ -297,9 +303,13 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext } meta.AddAnnotations(obj, map[string]string{ - v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), - "kubernetes.crossplane.io/provider-config-ref": cr.Spec.ProviderConfigReference.Name, + v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), }) + if c.kindObserver != nil { + meta.AddAnnotations(obj, map[string]string{ + AnnotationKeyProviderConfigRef: cr.Spec.ProviderConfigReference.Name, + }) + } if err := c.client.Create(ctx, obj); err != nil { return managed.ExternalCreation{}, errors.Wrap(err, errCreateObject) @@ -322,9 +332,13 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext } meta.AddAnnotations(obj, map[string]string{ - v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), - "kubernetes.crossplane.io/provider-config-ref": cr.Spec.ProviderConfigReference.Name, + v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), }) + if c.kindObserver != nil { + meta.AddAnnotations(obj, map[string]string{ + AnnotationKeyProviderConfigRef: cr.Spec.ProviderConfigReference.Name, + }) + } if err := c.client.Apply(ctx, obj); err != nil { return managed.ExternalUpdate{}, errors.Wrap(CleanErr(err), errApplyObject) @@ -475,7 +489,7 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) c.logger.Debug("Resolving referencies.") // Loop through references to resolve each referenced resource - var gvks []schema.GroupVersionKind + gvks := make([]schema.GroupVersionKind, 0, len(obj.Spec.References)) for _, ref := range obj.Spec.References { if ref.DependsOn == nil && ref.PatchesFrom == nil { continue @@ -512,8 +526,8 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) if c.kindObserver != nil { // Referenced resources always live on the control plane (i.e. local cluster), - // so we don't pass an extra rest config or provider config with the - // watch call. + // so we don't pass an extra rest config (defaulting local rest config) + // or provider config with the watch call. c.kindObserver.WatchResources(nil, "", gvks...) } diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 7e151bcd..814d685a 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -43,6 +43,7 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" kubernetesv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients" ) const ( @@ -238,7 +239,7 @@ func Test_connector_Connect(t *testing.T) { type args struct { client client.Client - clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) + clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) usage resource.Tracker mg resource.Managed } @@ -268,8 +269,12 @@ func Test_connector_Connect(t *testing.T) { }, "Success": { args: args{ - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { - return &test.MockClient{}, nil + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { + return &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{}, + }, + }, nil }, usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), mg: kubernetesObject(), @@ -280,7 +285,7 @@ func Test_connector_Connect(t *testing.T) { }, "ErrorGettingClientForProvider": { args: args{ - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { return nil, errBoom }, usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), @@ -309,7 +314,7 @@ func Test_connector_Connect(t *testing.T) { func Test_helmExternal_Observe(t *testing.T) { type args struct { - client resource.ClientApplicator + client clients.ClusterClient mg resource.Managed } type want struct { @@ -331,9 +336,11 @@ func Test_helmExternal_Observe(t *testing.T) { "NoKubernetesObjectExists": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(kerrors.NewNotFound(schema.GroupResource{}, "")), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(kerrors.NewNotFound(schema.GroupResource{}, "")), + }, }, }, }, @@ -355,9 +362,11 @@ func Test_helmExternal_Observe(t *testing.T) { "FailedToGet": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(errBoom), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(errBoom), + }, }, }, }, @@ -368,12 +377,14 @@ func Test_helmExternal_Observe(t *testing.T) { "NoLastAppliedAnnotation": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *externalResource() - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *externalResource() + return nil + }), + }, }, }, }, @@ -385,15 +396,17 @@ func Test_helmExternal_Observe(t *testing.T) { "NotUpToDate": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, - ) - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, + ) + return nil + }), + }, }, }, }, @@ -409,15 +422,17 @@ func Test_helmExternal_Observe(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace"}`, - ) - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace"}`, + ) + return nil + }), + }, }, }, }, @@ -433,12 +448,14 @@ func Test_helmExternal_Observe(t *testing.T) { "UpToDate": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + return nil + }), + }, }, }, }, @@ -457,12 +474,14 @@ func Test_helmExternal_Observe(t *testing.T) { obj.Spec.References = objectReferences() obj.Spec.References[0].PatchesFrom.FieldPath = ptr.To("nonexistent_field") }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + }), + }, }, }, }, @@ -477,9 +496,11 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = objectReferences() }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(errBoom), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(errBoom), + }, }, }, }, @@ -495,18 +516,20 @@ func Test_helmExternal_Observe(t *testing.T) { obj.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: time.Now()} obj.Spec.References = objectReferences() }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == testReferenceObjectName { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - } else if key.Name == externalResourceName { - return kerrors.NewNotFound(schema.GroupResource{}, "") - } - return errBoom + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key.Name == testReferenceObjectName { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + } else if key.Name == externalResourceName { + return kerrors.NewNotFound(schema.GroupResource{}, "") + } + return errBoom + }, + MockUpdate: test.NewMockUpdateFn(nil), }, - MockUpdate: test.NewMockUpdateFn(nil), }, }, }, @@ -520,19 +543,21 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = objectReferences() }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == testReferenceObjectName { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - } else if key.Name == externalResourceName { - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - return nil - } - return errBoom + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key.Name == testReferenceObjectName { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + } else if key.Name == externalResourceName { + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + return nil + } + return errBoom + }, + MockUpdate: test.NewMockUpdateFn(nil), }, - MockUpdate: test.NewMockUpdateFn(nil), }, }, }, @@ -550,9 +575,11 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = []v1alpha2.Reference{{}} }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil), + }, }, }, }, @@ -578,22 +605,24 @@ func Test_helmExternal_Observe(t *testing.T) { }, } }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - switch key.Name { - case externalResourceName: - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - case testSecretName: - *obj.(*unstructured.Unstructured) = unstructured.Unstructured{ - Object: map[string]interface{}{ - "data": map[string]interface{}{ - "db-password": "MTIzNDU=", + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + switch key.Name { + case externalResourceName: + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + case testSecretName: + *obj.(*unstructured.Unstructured) = unstructured.Unstructured{ + Object: map[string]interface{}{ + "data": map[string]interface{}{ + "db-password": "MTIzNDU=", + }, }, - }, + } } - } - return nil + return nil + }, }, }, }, @@ -624,16 +653,18 @@ func Test_helmExternal_Observe(t *testing.T) { }, } }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - switch key.Name { - case externalResourceName: - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - case testSecretName: - return errBoom - } - return nil + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + switch key.Name { + case externalResourceName: + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + case testSecretName: + return errBoom + } + return nil + }, }, }, }, @@ -647,15 +678,17 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.ManagementPolicies = xpv1.ManagementPolicies{xpv1.ManagementActionObserve} }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, - ) - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, + ) + return nil + }), + }, }, }, }, @@ -686,7 +719,7 @@ func Test_helmExternal_Observe(t *testing.T) { func Test_helmExternal_Create(t *testing.T) { type args struct { - client resource.ClientApplicator + client clients.ClusterClient mg resource.Managed } type want struct { @@ -718,9 +751,11 @@ func Test_helmExternal_Create(t *testing.T) { "FailedToCreate": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(errBoom), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(errBoom), + }, }, }, }, @@ -735,18 +770,20 @@ func Test_helmExternal_Create(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { - _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] - if !ok { - t.Errorf("Last applied annotation not set with create") - } - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { + _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] + if !ok { + t.Errorf("Last applied annotation not set with create") + } + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), + }, }, }, }, @@ -757,15 +794,17 @@ func Test_helmExternal_Create(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { - _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] - if !ok { - t.Errorf("Last applied annotation not set with create") - } - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { + _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] + if !ok { + t.Errorf("Last applied annotation not set with create") + } + return nil + }), + }, }, }, }, @@ -794,7 +833,7 @@ func Test_helmExternal_Create(t *testing.T) { func Test_helmExternal_Update(t *testing.T) { type args struct { - client resource.ClientApplicator + client clients.ClusterClient mg resource.Managed } type want struct { @@ -826,10 +865,12 @@ func Test_helmExternal_Update(t *testing.T) { "FailedToApply": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { - return errBoom - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { + return errBoom + }), + }, }, }, want: want{ @@ -843,13 +884,15 @@ func Test_helmExternal_Update(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(ctx context.Context, obj client.Object, op ...resource.ApplyOption) error { - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(ctx context.Context, obj client.Object, op ...resource.ApplyOption) error { + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), + }, }, }, want: want{ @@ -859,10 +902,12 @@ func Test_helmExternal_Update(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { + return nil + }), + }, }, }, want: want{ @@ -890,7 +935,7 @@ func Test_helmExternal_Update(t *testing.T) { func Test_helmExternal_Delete(t *testing.T) { type args struct { - client resource.ClientApplicator + client clients.ClusterClient mg resource.Managed } type want struct { @@ -921,9 +966,11 @@ func Test_helmExternal_Delete(t *testing.T) { "FailedToDelete": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(errBoom), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(errBoom), + }, }, }, }, @@ -938,14 +985,16 @@ func Test_helmExternal_Delete(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(nil, func(obj client.Object) error { - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(nil, func(obj client.Object) error { + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), + }, }, }, }, @@ -956,9 +1005,11 @@ func Test_helmExternal_Delete(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(nil), + client: &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(nil), + }, }, }, }, diff --git a/internal/controller/observedobjectcollection/reconciler_test.go b/internal/controller/observedobjectcollection/reconciler_test.go index f7faaa30..ad984cde 100644 --- a/internal/controller/observedobjectcollection/reconciler_test.go +++ b/internal/controller/observedobjectcollection/reconciler_test.go @@ -38,10 +38,12 @@ import ( xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" + "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients" ) func TestReconciler(t *testing.T) { @@ -380,8 +382,12 @@ func TestReconciler(t *testing.T) { r := &Reconciler{ client: tc.args.client, log: logging.NewNopLogger(), - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, error) { - return tc.args.client, nil + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { + return &clients.ApplicatorClientWithConfig{ + ClientApplicator: resource.ClientApplicator{ + Client: tc.args.client, + }, + }, nil }, observedObjectName: func(collection client.Object, matchedObject client.Object) (string, error) { return fmt.Sprintf("%s-%s", collection.GetName(), matchedObject.GetName()), nil From 5f12a80fe57b73544fc4c28e9f3f10a8e489ceeb Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 18:16:45 +0300 Subject: [PATCH 23/36] Start an informer per provider config & gvk pair Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/indexes.go | 19 +++--------- internal/controller/object/informers.go | 41 ++++++++++++------------- internal/controller/object/object.go | 20 +++--------- 3 files changed, 28 insertions(+), 52 deletions(-) diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index d127f81c..2ed9d634 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -110,12 +110,11 @@ func refKey(providerConfig, ns, name, kind, apiVersion string) string { func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { return func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { - config := getConfigName(ev.ObjectNew) - // "config" is the provider config name, which is the value for the - // provider config ref annotation. It will be empty for objects that - // are not controlled by the "Object" resource, i.e. referenced objects. + // "pc" is the provider pc name. It will be empty for referenced + // resources, as they are always on the control plane. + pc, _ := ctx.Value(keyProviderConfigName).(string) rGVK := ev.ObjectNew.GetObjectKind().GroupVersionKind() - key := refKey(config, ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) + key := refKey(pc, ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) objects := v1alpha2.ObjectList{} if err := ca.List(ctx, &objects, client.MatchingFields{resourceRefsIndex: key}); err != nil { @@ -124,16 +123,8 @@ func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx co } // queue those Objects for reconciliation for _, o := range objects.Items { - log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName()) + log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName(), "providerConfig", pc) q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}}) } } } - -func getConfigName(o client.Object) string { - a := o.GetAnnotations() - if a == nil { - return "" - } - return a[AnnotationKeyProviderConfigRef] -} diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index bcca7fde..85fadac0 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -2,7 +2,6 @@ package object import ( "context" - "github.com/crossplane/crossplane-runtime/pkg/errors" "strings" "sync" @@ -18,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/crossplane/crossplane-runtime/pkg/errors" "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" @@ -38,24 +38,22 @@ type resourceInformers struct { // resourceCaches holds the resource caches. These are dynamically started // and stopped based on the Objects that reference or managing them. - resourceCaches map[gvkWithHost]resourceCache + resourceCaches map[gvkWithConfig]resourceCache objectsCache cache.Cache - sink func(ev runtimeevent.UpdateEvent) + sink func(providerConfig string, ev runtimeevent.UpdateEvent) } -type gvkWithHost struct { - host string - gvk schema.GroupVersionKind +type gvkWithConfig struct { + // Which provider config was used to create this cache. We will use this + // information to figure out whether there are Objects relying on this cache + // left during garbage collection of caches. + providerConfig string + gvk schema.GroupVersionKind } type resourceCache struct { cache cache.Cache cancelFn context.CancelFunc - - // Which provider config was used to create this cache. We will use this - // information to figure out whether there are Objects relying on this cache - // left during garbage collection of caches. - providerConfig string } var _ source.Source = &resourceInformers{} @@ -69,13 +67,13 @@ func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q if i.sink != nil { return errors.New("source already started, cannot start it again") } - i.sink = func(ev runtimeevent.UpdateEvent) { + i.sink = func(providerConfig string, ev runtimeevent.UpdateEvent) { for _, p := range ps { if !p.Update(ev) { return } } - h.Update(ctx, ev, q) + h.Update(context.WithValue(ctx, keyProviderConfigName, providerConfig), ev, q) } go func() { @@ -108,11 +106,11 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin // start new informers for _, gvk := range gvks { - if _, found := i.resourceCaches[gvkWithHost{host: rc.Host, gvk: gvk}]; found { + if _, found := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}]; found { continue } - log := i.log.WithValues("host", rc.Host, "gvk", gvk.String()) + log := i.log.WithValues("providerConfig", providerConfig, "gvk", gvk.String()) ca, err := cache.New(rc, cache.Options{}) if err != nil { @@ -148,7 +146,8 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin ObjectOld: old, ObjectNew: obj, } - i.sink(ev) + + i.sink(providerConfig, ev) }, }); err != nil { cancelFn() @@ -163,11 +162,9 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin _ = ca.Start(ctx) }() - i.resourceCaches[gvkWithHost{host: rc.Host, gvk: gvk}] = resourceCache{ + i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] = resourceCache{ cache: ca, cancelFn: cancelFn, - - providerConfig: providerConfig, } // wait for in the background, and only when synced add to the routed cache @@ -189,8 +186,8 @@ func (i *resourceInformers) cleanupResourceInformers(ctx context.Context) { i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) for gh, ca := range i.resourceCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(ca.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) } if len(list.Items) > 0 { @@ -198,7 +195,7 @@ func (i *resourceInformers) cleanupResourceInformers(ctx context.Context) { } ca.cancelFn() - i.log.Info("Stopped resource watch", "provider config", ca.providerConfig, "host", gh.host, "gvk", gh.gvk) + i.log.Info("Stopped resource watch", "provider config", gh.providerConfig, "gvk", gh.gvk) delete(i.resourceCaches, gh) } } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 690ddf9a..d8ce3f3a 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -59,12 +59,10 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/internal/features" ) +type key int + const ( - // AnnotationKeyProviderConfigRef is the annotation key for the provider config - // reference on the managed kubernetes resource. We use this to trigger the - // reconciler for the Object with that provider config in case of watch - // events. Only used when alpha watches are enabled. - AnnotationKeyProviderConfigRef = "kubernetes.crossplane.io/provider-config-ref" + keyProviderConfigName key = iota ) const ( @@ -157,7 +155,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit config: mgr.GetConfig(), objectsCache: ca, - resourceCaches: make(map[gvkWithHost]resourceCache), + resourceCaches: make(map[gvkWithConfig]resourceCache), } conn.kindObserver = &i @@ -305,11 +303,6 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext meta.AddAnnotations(obj, map[string]string{ v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), }) - if c.kindObserver != nil { - meta.AddAnnotations(obj, map[string]string{ - AnnotationKeyProviderConfigRef: cr.Spec.ProviderConfigReference.Name, - }) - } if err := c.client.Create(ctx, obj); err != nil { return managed.ExternalCreation{}, errors.Wrap(err, errCreateObject) @@ -334,11 +327,6 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext meta.AddAnnotations(obj, map[string]string{ v1.LastAppliedConfigAnnotation: string(cr.Spec.ForProvider.Manifest.Raw), }) - if c.kindObserver != nil { - meta.AddAnnotations(obj, map[string]string{ - AnnotationKeyProviderConfigRef: cr.Spec.ProviderConfigReference.Name, - }) - } if err := c.client.Apply(ctx, obj); err != nil { return managed.ExternalUpdate{}, errors.Wrap(CleanErr(err), errApplyObject) From 1e56b1dc8c5bb590acbbdd6ab0e7a7bf931c04ba Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 2 May 2024 23:47:29 +0300 Subject: [PATCH 24/36] Start watching even if it does not exist (yet) Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/object.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index d8ce3f3a..8f34bab5 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -254,6 +254,10 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex if err != nil { return managed.ExternalObservation{}, err } + + if c.kindObserver != nil { + c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, desired.GroupVersionKind()) + } observed := desired.DeepCopy() @@ -270,12 +274,6 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex return managed.ExternalObservation{}, errors.Wrap(err, errGetObject) } - // We know the resource exists, so we can start watching it for realtime - // events if we have the kindObserver (i.e. watches enabled). - if c.kindObserver != nil { - c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, observed.GroupVersionKind()) - } - if err = c.setObserved(cr, observed); err != nil { return managed.ExternalObservation{}, err } From 139819043b3ba4323d7e04dab25908984bfd72fa Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 3 May 2024 08:54:14 +0300 Subject: [PATCH 25/36] Catch add or delete events as well Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/indexes.go | 10 ++++---- internal/controller/object/informers.go | 33 +++++++++++++++++++------ internal/controller/object/object.go | 4 +-- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index 2ed9d634..2bddca7f 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -108,13 +108,13 @@ func refKey(providerConfig, ns, name, kind, apiVersion string) string { return fmt.Sprintf("%s.%s.%s.%s.%s", providerConfig, name, ns, kind, apiVersion) } -func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { - return func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { +func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.GenericEvent, q workqueue.RateLimitingInterface) { + return func(ctx context.Context, ev runtimeevent.GenericEvent, q workqueue.RateLimitingInterface) { // "pc" is the provider pc name. It will be empty for referenced // resources, as they are always on the control plane. pc, _ := ctx.Value(keyProviderConfigName).(string) - rGVK := ev.ObjectNew.GetObjectKind().GroupVersionKind() - key := refKey(pc, ev.ObjectNew.GetNamespace(), ev.ObjectNew.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) + rGVK := ev.Object.GetObjectKind().GroupVersionKind() + key := refKey(pc, ev.Object.GetNamespace(), ev.Object.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) objects := v1alpha2.ObjectList{} if err := ca.List(ctx, &objects, client.MatchingFields{resourceRefsIndex: key}); err != nil { @@ -123,7 +123,7 @@ func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx co } // queue those Objects for reconciliation for _, o := range objects.Items { - log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.ObjectNew.GetName(), "providerConfig", pc) + log.Info("Enqueueing Object because referenced resource changed", "name", o.GetName(), "referencedGVK", rGVK.String(), "referencedName", ev.Object.GetName(), "providerConfig", pc) q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: o.GetName()}}) } } diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 85fadac0..26367fcf 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -40,7 +40,7 @@ type resourceInformers struct { // and stopped based on the Objects that reference or managing them. resourceCaches map[gvkWithConfig]resourceCache objectsCache cache.Cache - sink func(providerConfig string, ev runtimeevent.UpdateEvent) + sink func(providerConfig string, ev runtimeevent.GenericEvent) } type gvkWithConfig struct { @@ -67,13 +67,13 @@ func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q if i.sink != nil { return errors.New("source already started, cannot start it again") } - i.sink = func(providerConfig string, ev runtimeevent.UpdateEvent) { + i.sink = func(providerConfig string, ev runtimeevent.GenericEvent) { for _, p := range ps { - if !p.Update(ev) { + if !p.Generic(ev) { return } } - h.Update(context.WithValue(ctx, keyProviderConfigName, providerConfig), ev, q) + h.Generic(context.WithValue(ctx, keyProviderConfigName, providerConfig), ev, q) } go func() { @@ -132,6 +132,16 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin } if _, err := inf.AddEventHandler(kcache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + i.lock.RLock() + defer i.lock.RUnlock() + + ev := runtimeevent.GenericEvent{ + Object: obj.(client.Object), + } + + i.sink(providerConfig, ev) + }, UpdateFunc: func(oldObj, newObj interface{}) { old := oldObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. obj := newObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. @@ -142,9 +152,18 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin i.lock.RLock() defer i.lock.RUnlock() - ev := runtimeevent.UpdateEvent{ - ObjectOld: old, - ObjectNew: obj, + ev := runtimeevent.GenericEvent{ + Object: obj, + } + + i.sink(providerConfig, ev) + }, + DeleteFunc: func(obj interface{}) { + i.lock.RLock() + defer i.lock.RUnlock() + + ev := runtimeevent.GenericEvent{ + Object: obj.(client.Object), } i.sink(providerConfig, ev) diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 8f34bab5..ede6c134 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -168,7 +168,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit } cb = cb.WatchesRawSource(&i, handler.Funcs{ - UpdateFunc: func(ctx context.Context, ev runtimeevent.UpdateEvent, q workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, ev runtimeevent.GenericEvent, q workqueue.RateLimitingInterface) { enqueueObjectsForReferences(ca, l)(ctx, ev, q) }, }) @@ -254,7 +254,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex if err != nil { return managed.ExternalObservation{}, err } - + if c.kindObserver != nil { c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, desired.GroupVersionKind()) } From 9714e9e15a81ea04df0ef68c5d950bbf02af9242 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 3 May 2024 12:18:22 +0300 Subject: [PATCH 26/36] Stop watching on Object deletion Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/informers.go | 85 ++++++++++++++++--------- internal/controller/object/object.go | 28 +++++++- 2 files changed, 81 insertions(+), 32 deletions(-) diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 26367fcf..ce93064b 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -31,16 +31,15 @@ import ( // resources, and inform resourceInformers about them via the // WatchReferencedResources method. type resourceInformers struct { - log logging.Logger - config *rest.Config + log logging.Logger + config *rest.Config + objectsCache cache.Cache + sink func(providerConfig string, ev runtimeevent.GenericEvent) lock sync.RWMutex // everything below is protected by this lock - // resourceCaches holds the resource caches. These are dynamically started // and stopped based on the Objects that reference or managing them. resourceCaches map[gvkWithConfig]resourceCache - objectsCache cache.Cache - sink func(providerConfig string, ev runtimeevent.GenericEvent) } type gvkWithConfig struct { @@ -62,8 +61,6 @@ var _ source.Source = &resourceInformers{} // source with h as the sink of update events. It keeps sending events until // ctx is done. func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, ps ...predicate.Predicate) error { - i.lock.Lock() - defer i.lock.Unlock() if i.sink != nil { return errors.New("source already started, cannot start it again") } @@ -78,9 +75,6 @@ func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q go func() { <-ctx.Done() - - i.lock.Lock() - defer i.lock.Unlock() i.sink = nil }() @@ -93,20 +87,20 @@ func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q // every reconcile to make resourceInformers aware of the referenced or managed // resources of the given Object. // -// Note that this complements cleanupResourceInformers which regularly +// Note that this complements garbageCollectResourceInformers which regularly // garbage collects resource informers that are no longer referenced by // any Object. func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { - i.lock.RLock() - defer i.lock.RUnlock() - if rc == nil { rc = i.config } // start new informers for _, gvk := range gvks { - if _, found := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}]; found { + i.lock.RLock() + _, found := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] + i.lock.RUnlock() + if found { continue } @@ -133,9 +127,6 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin if _, err := inf.AddEventHandler(kcache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - i.lock.RLock() - defer i.lock.RUnlock() - ev := runtimeevent.GenericEvent{ Object: obj.(client.Object), } @@ -149,9 +140,6 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin return } - i.lock.RLock() - defer i.lock.RUnlock() - ev := runtimeevent.GenericEvent{ Object: obj, } @@ -159,9 +147,6 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin i.sink(providerConfig, ev) }, DeleteFunc: func(obj interface{}) { - i.lock.RLock() - defer i.lock.RUnlock() - ev := runtimeevent.GenericEvent{ Object: obj.(client.Object), } @@ -181,10 +166,12 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin _ = ca.Start(ctx) }() + i.lock.Lock() i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] = resourceCache{ cache: ca, cancelFn: cancelFn, } + i.lock.Unlock() // wait for in the background, and only when synced add to the routed cache go func() { @@ -195,12 +182,50 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin } } -// cleanupResourceInformers garbage collects resource informers that are no -// longer referenced by any Object. -// -// Note that this complements WatchResources which starts informers for -// the resources referenced or managed by an Object. -func (i *resourceInformers) cleanupResourceInformers(ctx context.Context) { +func (i *resourceInformers) StopWatchingResources(ctx context.Context, providerConfig string, gvks ...schema.GroupVersionKind) { + i.lock.Lock() + defer i.lock.Unlock() + + for _, gvk := range gvks { + ca, found := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] + if !found { + continue + } + // Check if there are any other objects referencing this resource GVK. + list := v1alpha2.ObjectList{} + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(providerConfig, gvk.Kind, gvk.Group, gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(providerConfig, gvk.Kind, gvk.Group, gvk.Version)) + } + + inUse := false + for _, o := range list.Items { + // We only care about objects that are not being deleted. Otherwise, + // we are getting into deadlocks while stopping the watches during + // deletion. + if o.GetDeletionTimestamp().IsZero() { + inUse = true + break + } + } + if inUse { + continue + } + + ca.cancelFn() + i.log.Info("Stopped resource watch", "provider config", providerConfig, "gvk", gvk) + delete(i.resourceCaches, gvkWithConfig{providerConfig: providerConfig, gvk: gvk}) + } +} + +// garbageCollectResourceInformers garbage collects resource informers that are +// no longer referenced by any Object. Ideally, all resource informers should +// stopped/cleaned up when the Object is deleted. However, in practice, this +// is not always the case. This method is a safety net to clean up resource +// informers that are no longer referenced by any Object. +func (i *resourceInformers) garbageCollectResourceInformers(ctx context.Context) { + i.lock.Lock() + defer i.lock.Unlock() + // stop old informers i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) for gh, ca := range i.resourceCaches { diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index ede6c134..90422bdf 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -102,6 +102,10 @@ type KindObserver interface { // WatchResources starts a watch of the given kinds to trigger reconciles // when a referenced or managed objects of those kinds changes. WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) + + // UnwatchResources stops watching the given kinds if they are no longer + // referenced or managed by any other Object that is not being deleted. + StopWatchingResources(ctx context.Context, providerConfig string, gvks ...schema.GroupVersionKind) } // Setup adds a controller that reconciles Object managed resources. @@ -160,8 +164,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit conn.kindObserver = &i if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - // Run every 5 minutes. - wait.UntilWithContext(ctx, i.cleanupResourceInformers, 5*time.Minute) + wait.UntilWithContext(ctx, i.garbageCollectResourceInformers, time.Minute) return nil })); err != nil { return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") @@ -346,6 +349,27 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error { return err } + if c.kindObserver != nil { + c.kindObserver.StopWatchingResources(ctx, cr.Spec.ProviderConfigReference.Name, obj.GroupVersionKind()) + if len(cr.Spec.References) > 0 { + gvks := make([]schema.GroupVersionKind, 0, len(cr.Spec.References)) + for _, ref := range cr.Spec.References { + if ref.DependsOn == nil && ref.PatchesFrom == nil { + continue + } + + refAPIVersion, refKind, _, _ := getReferenceInfo(ref) + g, v := parseAPIVersion(refAPIVersion) + gvks = append(gvks, schema.GroupVersionKind{ + Group: g, + Version: v, + Kind: refKind, + }) + } + c.kindObserver.StopWatchingResources(ctx, "", gvks...) + } + } + return errors.Wrap(resource.IgnoreNotFound(c.client.Delete(ctx, obj)), errDeleteObject) } From d1ccd448854a709ffbcf31e6fa0955261b39fb6b Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 3 May 2024 14:55:17 +0300 Subject: [PATCH 27/36] Only watch the resource if desired Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- apis/object/v1alpha1/types.go | 8 ++++++++ apis/object/v1alpha1/zz_generated.deepcopy.go | 5 +++++ apis/object/v1alpha2/types.go | 8 ++++++++ apis/object/v1alpha2/zz_generated.deepcopy.go | 5 +++++ examples/object/object.yaml | 4 ++++ .../references/patches-from-resource.yaml | 4 ++++ internal/controller/object/object.go | 11 +++++++--- .../kubernetes.crossplane.io_objects.yaml | 20 +++++++++++++++++++ 8 files changed, 62 insertions(+), 3 deletions(-) diff --git a/apis/object/v1alpha1/types.go b/apis/object/v1alpha1/types.go index d44eca82..76f65f55 100644 --- a/apis/object/v1alpha1/types.go +++ b/apis/object/v1alpha1/types.go @@ -124,6 +124,14 @@ type ObjectSpec struct { ManagementPolicy `json:"managementPolicy,omitempty"` References []Reference `json:"references,omitempty"` Readiness Readiness `json:"readiness,omitempty"` + // Watch enables watching the referenced or managed kubernetes resources. + // + // THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored + // unless the relevant Crossplane feature flag is enabled, and may be + // changed or removed without notice. + // +optional + // +kubebuilder:default=false + Watch *bool `json:"watch,omitempty"` } // ReadinessPolicy defines how the Object's readiness condition should be computed. diff --git a/apis/object/v1alpha1/zz_generated.deepcopy.go b/apis/object/v1alpha1/zz_generated.deepcopy.go index aad34718..b0968cc2 100644 --- a/apis/object/v1alpha1/zz_generated.deepcopy.go +++ b/apis/object/v1alpha1/zz_generated.deepcopy.go @@ -165,6 +165,11 @@ func (in *ObjectSpec) DeepCopyInto(out *ObjectSpec) { } } out.Readiness = in.Readiness + if in.Watch != nil { + in, out := &in.Watch, &out.Watch + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSpec. diff --git a/apis/object/v1alpha2/types.go b/apis/object/v1alpha2/types.go index 75210ed8..fb8fb136 100644 --- a/apis/object/v1alpha2/types.go +++ b/apis/object/v1alpha2/types.go @@ -97,6 +97,14 @@ type ObjectSpec struct { ForProvider ObjectParameters `json:"forProvider"` References []Reference `json:"references,omitempty"` Readiness Readiness `json:"readiness,omitempty"` + // Watch enables watching the referenced or managed kubernetes resources. + // + // THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored + // unless the relevant Crossplane feature flag is enabled, and may be + // changed or removed without notice. + // +optional + // +kubebuilder:default=false + Watch *bool `json:"watch,omitempty"` } // ReadinessPolicy defines how the Object's readiness condition should be computed. diff --git a/apis/object/v1alpha2/zz_generated.deepcopy.go b/apis/object/v1alpha2/zz_generated.deepcopy.go index c0d78570..ef2e20e8 100644 --- a/apis/object/v1alpha2/zz_generated.deepcopy.go +++ b/apis/object/v1alpha2/zz_generated.deepcopy.go @@ -164,6 +164,11 @@ func (in *ObjectSpec) DeepCopyInto(out *ObjectSpec) { } } out.Readiness = in.Readiness + if in.Watch != nil { + in, out := &in.Watch, &out.Watch + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSpec. diff --git a/examples/object/object.yaml b/examples/object/object.yaml index 255e5b7b..b028dd23 100644 --- a/examples/object/object.yaml +++ b/examples/object/object.yaml @@ -3,6 +3,10 @@ kind: Object metadata: name: sample-namespace spec: + # Watch for changes to the Namespace object. + # Watching resources is an alpha feature and needs to be enabled with --enable-watches + # in the provider to get this configuration working. + # watch: true forProvider: manifest: apiVersion: v1 diff --git a/examples/object/references/patches-from-resource.yaml b/examples/object/references/patches-from-resource.yaml index 843aaf3d..3f06366b 100644 --- a/examples/object/references/patches-from-resource.yaml +++ b/examples/object/references/patches-from-resource.yaml @@ -4,6 +4,10 @@ kind: Object metadata: name: foo spec: + # Watch for changes to the Namespace object. + # Watching resources is an alpha feature and needs to be enabled with --enable-watches + # in the provider to get this configuration working. + # watch: true references: # Use patchesFrom to patch field from other k8s resource to this object - patchesFrom: diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 90422bdf..00b6b8e0 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" @@ -258,7 +259,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex return managed.ExternalObservation{}, err } - if c.kindObserver != nil { + if c.shouldWatch(cr) { c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, desired.GroupVersionKind()) } @@ -349,7 +350,7 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error { return err } - if c.kindObserver != nil { + if c.shouldWatch(cr) { c.kindObserver.StopWatchingResources(ctx, cr.Spec.ProviderConfigReference.Name, obj.GroupVersionKind()) if len(cr.Spec.References) > 0 { gvks := make([]schema.GroupVersionKind, 0, len(cr.Spec.References)) @@ -534,7 +535,7 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) }) } - if c.kindObserver != nil { + if c.shouldWatch(obj) { // Referenced resources always live on the control plane (i.e. local cluster), // so we don't pass an extra rest config (defaulting local rest config) // or provider config with the watch call. @@ -715,6 +716,10 @@ func connectionDetails(ctx context.Context, kube client.Client, connDetails []v1 return mcd, nil } +func (c *external) shouldWatch(cr *v1alpha2.Object) bool { + return c.kindObserver != nil && ptr.Deref(cr.Spec.Watch, false) +} + func unstructuredFromObjectRef(r v1.ObjectReference) unstructured.Unstructured { u := unstructured.Unstructured{} u.SetAPIVersion(r.APIVersion) diff --git a/package/crds/kubernetes.crossplane.io_objects.yaml b/package/crds/kubernetes.crossplane.io_objects.yaml index 4a652679..f3aded13 100644 --- a/package/crds/kubernetes.crossplane.io_objects.yaml +++ b/package/crds/kubernetes.crossplane.io_objects.yaml @@ -400,6 +400,16 @@ spec: type: string type: object type: array + watch: + default: false + description: |- + Watch enables watching the referenced or managed kubernetes resources. + + + THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored + unless the relevant Crossplane feature flag is enabled, and may be + changed or removed without notice. + type: boolean writeConnectionSecretToRef: description: |- WriteConnectionSecretToReference specifies the namespace and name of a @@ -834,6 +844,16 @@ spec: type: string type: object type: array + watch: + default: false + description: |- + Watch enables watching the referenced or managed kubernetes resources. + + + THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored + unless the relevant Crossplane feature flag is enabled, and may be + changed or removed without notice. + type: boolean writeConnectionSecretToRef: description: |- WriteConnectionSecretToReference specifies the namespace and name of a From bd6d7ee5afc4645749fcc6f929cf88ac4481961a Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Sat, 4 May 2024 09:45:27 +0300 Subject: [PATCH 28/36] Better naming for index functions Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/indexes.go | 40 ++++++++++++------------- internal/controller/object/informers.go | 24 ++++++++++++--- internal/controller/object/object.go | 4 +-- internal/features/features.go | 23 +++++++------- 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/internal/controller/object/indexes.go b/internal/controller/object/indexes.go index 2bddca7f..25f28283 100644 --- a/internal/controller/object/indexes.go +++ b/internal/controller/object/indexes.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The Crossplane Authors. +Copyright 2024 The Crossplane Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -42,13 +42,14 @@ const ( ) var ( - _ client.IndexerFunc = IndexReferencedResourceRefGVKs - _ client.IndexerFunc = IndexReferencesResourcesRefs + _ client.IndexerFunc = IndexByProviderGVK + _ client.IndexerFunc = IndexByProviderNamespacedNameGVK ) -// IndexReferencedResourceRefGVKs assumes the passed object is an Object. It -// returns gvk keys for every resource referenced or managed by the Object. -func IndexReferencedResourceRefGVKs(o client.Object) []string { +// IndexByProviderGVK assumes the passed object is an Object. It returns keys +// with "ProviderConfig + GVK" for every resource referenced or managed by the +// Object. +func IndexByProviderGVK(o client.Object) []string { obj, ok := o.(*v1alpha2.Object) if !ok { return nil // should never happen @@ -60,27 +61,28 @@ func IndexReferencedResourceRefGVKs(o client.Object) []string { for _, ref := range refs { refAPIVersion, refKind, _, _ := getReferenceInfo(ref) group, version := parseAPIVersion(refAPIVersion) - // References are always on control plane, so we don't pass the config name. - keys = append(keys, refKeyGKV("", refKind, group, version)) + providerConfig := "" // references are always local (i.e. on the control plane), which we represent as an empty provider config. + keys = append(keys, refKeyProviderGVK(providerConfig, refKind, group, version)) } // Index the desired object. // We don't expect errors here, as the getDesired function is already called // in the reconciler and the desired object already validated. d, _ := getDesired(obj) - keys = append(keys, refKeyGKV(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer. + keys = append(keys, refKeyProviderGVK(obj.Spec.ProviderConfigReference.Name, d.GetKind(), d.GroupVersionKind().Group, d.GroupVersionKind().Version)) // unification is done by the informer. // unification is done by the informer. return keys } -func refKeyGKV(providerConfig, kind, group, version string) string { +func refKeyProviderGVK(providerConfig, kind, group, version string) string { return fmt.Sprintf("%s.%s.%s.%s", providerConfig, kind, group, version) } -// IndexReferencesResourcesRefs assumes the passed object is an Object. It -// returns keys for every resource referenced or managed by the Object. -func IndexReferencesResourcesRefs(o client.Object) []string { +// IndexByProviderNamespacedNameGVK assumes the passed object is an Object. It +// returns keys with "ProviderConfig + NamespacedName + GVK" for every resource +// referenced or managed by the Object. +func IndexByProviderNamespacedNameGVK(o client.Object) []string { obj, ok := o.(*v1alpha2.Object) if !ok { return nil // should never happen @@ -91,30 +93,28 @@ func IndexReferencesResourcesRefs(o client.Object) []string { keys := make([]string, 0, len(refs)) for _, ref := range refs { refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) - // References are always on control plane, so we don't pass the provider config name. - keys = append(keys, refKey("", refNamespace, refName, refKind, refAPIVersion)) + providerConfig := "" // references are always local (i.e. on the control plane), which we represent as an empty provider config. + keys = append(keys, refKeyProviderNamespacedNameGVK(providerConfig, refNamespace, refName, refKind, refAPIVersion)) } // Index the desired object. // We don't expect errors here, as the getDesired function is already called // in the reconciler and the desired object already validated. d, _ := getDesired(obj) - keys = append(keys, refKey(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer. + keys = append(keys, refKeyProviderNamespacedNameGVK(obj.Spec.ProviderConfigReference.Name, d.GetNamespace(), d.GetName(), d.GetKind(), d.GetAPIVersion())) // unification is done by the informer. return keys } -func refKey(providerConfig, ns, name, kind, apiVersion string) string { +func refKeyProviderNamespacedNameGVK(providerConfig, ns, name, kind, apiVersion string) string { return fmt.Sprintf("%s.%s.%s.%s.%s", providerConfig, name, ns, kind, apiVersion) } func enqueueObjectsForReferences(ca cache.Cache, log logging.Logger) func(ctx context.Context, ev runtimeevent.GenericEvent, q workqueue.RateLimitingInterface) { return func(ctx context.Context, ev runtimeevent.GenericEvent, q workqueue.RateLimitingInterface) { - // "pc" is the provider pc name. It will be empty for referenced - // resources, as they are always on the control plane. pc, _ := ctx.Value(keyProviderConfigName).(string) rGVK := ev.Object.GetObjectKind().GroupVersionKind() - key := refKey(pc, ev.Object.GetNamespace(), ev.Object.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) + key := refKeyProviderNamespacedNameGVK(pc, ev.Object.GetNamespace(), ev.Object.GetName(), rGVK.Kind, rGVK.GroupVersion().String()) objects := v1alpha2.ObjectList{} if err := ca.List(ctx, &objects, client.MatchingFields{resourceRefsIndex: key}); err != nil { diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index ce93064b..af3a76c7 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package object import ( @@ -193,8 +209,8 @@ func (i *resourceInformers) StopWatchingResources(ctx context.Context, providerC } // Check if there are any other objects referencing this resource GVK. list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(providerConfig, gvk.Kind, gvk.Group, gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(providerConfig, gvk.Kind, gvk.Group, gvk.Version)) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)) } inUse := false @@ -230,8 +246,8 @@ func (i *resourceInformers) garbageCollectResourceInformers(ctx context.Context) i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) for gh, ca := range i.resourceCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyGKV(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyGKV(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) } if len(list.Items) > 0 { diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 00b6b8e0..769d7c98 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -148,10 +148,10 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit if o.Features.Enabled(features.EnableAlphaWatches) { ca := mgr.GetCache() - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefGVKsIndex, IndexReferencedResourceRefGVKs); err != nil { + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefGVKsIndex, IndexByProviderGVK); err != nil { return errors.Wrap(err, "cannot add index for object reference GVKs") } - if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefsIndex, IndexReferencesResourcesRefs); err != nil { + if err := ca.IndexField(context.Background(), &v1alpha2.Object{}, resourceRefsIndex, IndexByProviderNamespacedNameGVK); err != nil { return errors.Wrap(err, "cannot add index for object references") } diff --git a/internal/features/features.go b/internal/features/features.go index 3c294983..41b4a4bf 100644 --- a/internal/features/features.go +++ b/internal/features/features.go @@ -1,14 +1,17 @@ /* - Copyright 2022 The Crossplane Authors. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Copyright 2024 The Crossplane Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ package features From 4b863a746423578ecc140b3a447674fd89b9eb6e Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 9 May 2024 15:29:15 +0300 Subject: [PATCH 29/36] Do not use pointer for optional bool watch Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- apis/object/v1alpha1/types.go | 2 +- apis/object/v1alpha1/zz_generated.deepcopy.go | 5 ----- apis/object/v1alpha2/types.go | 2 +- apis/object/v1alpha2/zz_generated.deepcopy.go | 5 ----- internal/controller/object/object.go | 3 +-- 5 files changed, 3 insertions(+), 14 deletions(-) diff --git a/apis/object/v1alpha1/types.go b/apis/object/v1alpha1/types.go index 76f65f55..dfed2914 100644 --- a/apis/object/v1alpha1/types.go +++ b/apis/object/v1alpha1/types.go @@ -131,7 +131,7 @@ type ObjectSpec struct { // changed or removed without notice. // +optional // +kubebuilder:default=false - Watch *bool `json:"watch,omitempty"` + Watch bool `json:"watch,omitempty"` } // ReadinessPolicy defines how the Object's readiness condition should be computed. diff --git a/apis/object/v1alpha1/zz_generated.deepcopy.go b/apis/object/v1alpha1/zz_generated.deepcopy.go index b0968cc2..aad34718 100644 --- a/apis/object/v1alpha1/zz_generated.deepcopy.go +++ b/apis/object/v1alpha1/zz_generated.deepcopy.go @@ -165,11 +165,6 @@ func (in *ObjectSpec) DeepCopyInto(out *ObjectSpec) { } } out.Readiness = in.Readiness - if in.Watch != nil { - in, out := &in.Watch, &out.Watch - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSpec. diff --git a/apis/object/v1alpha2/types.go b/apis/object/v1alpha2/types.go index fb8fb136..426def82 100644 --- a/apis/object/v1alpha2/types.go +++ b/apis/object/v1alpha2/types.go @@ -104,7 +104,7 @@ type ObjectSpec struct { // changed or removed without notice. // +optional // +kubebuilder:default=false - Watch *bool `json:"watch,omitempty"` + Watch bool `json:"watch,omitempty"` } // ReadinessPolicy defines how the Object's readiness condition should be computed. diff --git a/apis/object/v1alpha2/zz_generated.deepcopy.go b/apis/object/v1alpha2/zz_generated.deepcopy.go index ef2e20e8..c0d78570 100644 --- a/apis/object/v1alpha2/zz_generated.deepcopy.go +++ b/apis/object/v1alpha2/zz_generated.deepcopy.go @@ -164,11 +164,6 @@ func (in *ObjectSpec) DeepCopyInto(out *ObjectSpec) { } } out.Readiness = in.Readiness - if in.Watch != nil { - in, out := &in.Watch, &out.Watch - *out = new(bool) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSpec. diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 769d7c98..2d8d4e32 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" runtimeevent "sigs.k8s.io/controller-runtime/pkg/event" @@ -717,7 +716,7 @@ func connectionDetails(ctx context.Context, kube client.Client, connDetails []v1 } func (c *external) shouldWatch(cr *v1alpha2.Object) bool { - return c.kindObserver != nil && ptr.Deref(cr.Spec.Watch, false) + return c.kindObserver != nil && cr.Spec.Watch } func unstructuredFromObjectRef(r v1.ObjectReference) unstructured.Unstructured { From cbeeeff296708ec2f9adfeb6b58c9ef19f7ecaf9 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Thu, 9 May 2024 22:25:24 +0300 Subject: [PATCH 30/36] Double check if cache is started already Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/informers.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index af3a76c7..b38fe235 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -183,13 +183,21 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin }() i.lock.Lock() + _, ok := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] + if ok { + // Another goroutine already started the cache in parallel. We + // should cancel the new one. + cancelFn() + i.lock.Unlock() + continue + } i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] = resourceCache{ cache: ca, cancelFn: cancelFn, } i.lock.Unlock() - // wait for in the background, and only when synced add to the routed cache + // wait for in the background. go func() { if synced := ca.WaitForCacheSync(ctx); synced { log.Debug("Resource cache synced") @@ -211,6 +219,7 @@ func (i *resourceInformers) StopWatchingResources(ctx context.Context, providerC list := v1alpha2.ObjectList{} if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)}); err != nil { i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)) + continue } inUse := false @@ -248,6 +257,7 @@ func (i *resourceInformers) garbageCollectResourceInformers(ctx context.Context) list := v1alpha2.ObjectList{} if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) + continue } if len(list.Items) > 0 { From 2f543db5cc206f145b26a8eac7df660c79ac63de Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 10 May 2024 12:35:09 +0300 Subject: [PATCH 31/36] Refactor clients package Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/clients/clients.go | 198 +-------- internal/clients/kube/kube.go | 175 ++++++++ internal/controller/object/object.go | 23 +- internal/controller/object/object_test.go | 406 ++++++++---------- .../observedobjectcollection/reconciler.go | 9 +- .../reconciler_test.go | 11 +- 6 files changed, 370 insertions(+), 452 deletions(-) create mode 100644 internal/clients/kube/kube.go diff --git a/internal/clients/clients.go b/internal/clients/clients.go index 34b22d30..2a31392e 100644 --- a/internal/clients/clients.go +++ b/internal/clients/clients.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Crossplane Authors. +Copyright 2024 The Crossplane Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -12,199 +12,3 @@ limitations under the License. */ package clients - -import ( - "context" - - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/clientcmd/api" - "sigs.k8s.io/controller-runtime/pkg/client" - - xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" - "github.com/crossplane/crossplane-runtime/pkg/resource" - - "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients/azure" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients/gke" -) - -const ( - errGetPC = "cannot get ProviderConfig" - errGetCreds = "cannot get credentials" - errCreateRestConfig = "cannot create new REST config using provider secret" - errExtractGoogleCredentials = "cannot extract Google Application Credentials" - errInjectGoogleCredentials = "cannot wrap REST client with Google Application Credentials" - errExtractAzureCredentials = "failed to extract Azure Application Credentials" - errInjectAzureCredentials = "failed to wrap REST client with Azure Application Credentials" -) - -// NewRESTConfig returns a rest config given a secret with connection information. -func NewRESTConfig(kubeconfig []byte) (*rest.Config, error) { - ac, err := clientcmd.Load(kubeconfig) - if err != nil { - return nil, errors.Wrap(err, "failed to load kubeconfig") - } - return restConfigFromAPIConfig(ac) -} - -// NewKubeClient returns a kubernetes client given a secret with connection -// information. -func NewKubeClient(config *rest.Config) (client.Client, error) { - kc, err := client.New(config, client.Options{}) - if err != nil { - return nil, errors.Wrap(err, "cannot create Kubernetes client") - } - - return kc, nil -} - -func restConfigFromAPIConfig(c *api.Config) (*rest.Config, error) { - if c.CurrentContext == "" { - return nil, errors.New("currentContext not set in kubeconfig") - } - ctx := c.Contexts[c.CurrentContext] - cluster := c.Clusters[ctx.Cluster] - if cluster == nil { - return nil, errors.Errorf("cluster for currentContext (%s) not found", c.CurrentContext) - } - user := c.AuthInfos[ctx.AuthInfo] - if user == nil { - // We don't require a user because it's possible user - // authorization configuration will be loaded from a separate - // set of identity credentials (e.g. Google Application Creds). - user = &api.AuthInfo{} - } - config := &rest.Config{ - Host: cluster.Server, - Username: user.Username, - Password: user.Password, - BearerToken: user.Token, - BearerTokenFile: user.TokenFile, - Impersonate: rest.ImpersonationConfig{ - UserName: user.Impersonate, - Groups: user.ImpersonateGroups, - Extra: user.ImpersonateUserExtra, - }, - AuthProvider: user.AuthProvider, - ExecProvider: user.Exec, - TLSClientConfig: rest.TLSClientConfig{ - Insecure: cluster.InsecureSkipTLSVerify, - ServerName: cluster.TLSServerName, - CertData: user.ClientCertificateData, - KeyData: user.ClientKeyData, - CAData: cluster.CertificateAuthorityData, - }, - } - - // NOTE(tnthornton): these values match the burst and QPS values in kubectl. - // xref: https://github.com/kubernetes/kubernetes/pull/105520 - config.Burst = 300 - config.QPS = 50 - - return config, nil -} - -// RestConfigGetter is an interface that provides a REST config. -type RestConfigGetter interface { - // GetConfig returns an initialized Config - GetConfig() *rest.Config -} - -// ClusterClient is a client that can be used to interact with a Kubernetes -// cluster including getting its rest config. -type ClusterClient interface { - client.Client - resource.Applicator - RestConfigGetter -} - -// ApplicatorClientWithConfig is a ClusterClient that also has a rest config. -type ApplicatorClientWithConfig struct { - resource.ClientApplicator - config *rest.Config -} - -// GetConfig returns the rest config for the client. -func (c *ApplicatorClientWithConfig) GetConfig() *rest.Config { - return c.config -} - -// ClientForProvider returns the client for the given provider config -func ClientForProvider(ctx context.Context, inclusterClient client.Client, providerConfigName string) (ClusterClient, error) { //nolint:gocyclo - pc := &v1alpha1.ProviderConfig{} - if err := inclusterClient.Get(ctx, types.NamespacedName{Name: providerConfigName}, pc); err != nil { - return nil, errors.Wrap(err, errGetPC) - } - - var rc *rest.Config - var err error - - switch cd := pc.Spec.Credentials; cd.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - rc, err = rest.InClusterConfig() - if err != nil { - return nil, errors.Wrap(err, errCreateRestConfig) - } - default: - kc, err := resource.CommonCredentialExtractor(ctx, cd.Source, inclusterClient, cd.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errGetCreds) - } - - if rc, err = NewRESTConfig(kc); err != nil { - return nil, errors.Wrap(err, errCreateRestConfig) - } - } - - if id := pc.Spec.Identity; id != nil { - switch id.Type { - case v1alpha1.IdentityTypeGoogleApplicationCredentials: - switch id.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - if err := gke.WrapRESTConfig(ctx, rc, nil, gke.DefaultScopes...); err != nil { - return nil, errors.Wrap(err, errInjectGoogleCredentials) - } - default: - creds, err := resource.CommonCredentialExtractor(ctx, id.Source, inclusterClient, id.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errExtractGoogleCredentials) - } - - if err := gke.WrapRESTConfig(ctx, rc, creds, gke.DefaultScopes...); err != nil { - return nil, errors.Wrap(err, errInjectGoogleCredentials) - } - } - case v1alpha1.IdentityTypeAzureServicePrincipalCredentials: - switch id.Source { //nolint:exhaustive - case xpv1.CredentialsSourceInjectedIdentity: - return nil, errors.Errorf("%s is not supported as identity source for identity type %s", - xpv1.CredentialsSourceInjectedIdentity, v1alpha1.IdentityTypeAzureServicePrincipalCredentials) - default: - creds, err := resource.CommonCredentialExtractor(ctx, id.Source, inclusterClient, id.CommonCredentialSelectors) - if err != nil { - return nil, errors.Wrap(err, errExtractAzureCredentials) - } - - if err := azure.WrapRESTConfig(ctx, rc, creds); err != nil { - return nil, errors.Wrap(err, errInjectAzureCredentials) - } - } - default: - return nil, errors.Errorf("unknown identity type: %s", id.Type) - } - } - k, err := NewKubeClient(rc) - if err != nil { - return nil, errors.Wrap(err, "cannot create Kubernetes client") - } - return &ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: k, - Applicator: resource.NewAPIPatchingApplicator(k), - }, - config: rc, - }, nil -} diff --git a/internal/clients/kube/kube.go b/internal/clients/kube/kube.go new file mode 100644 index 00000000..f7420045 --- /dev/null +++ b/internal/clients/kube/kube.go @@ -0,0 +1,175 @@ +/* +Copyright 2024 The Crossplane Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "context" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/controller-runtime/pkg/client" + + xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" + "github.com/crossplane/crossplane-runtime/pkg/resource" + + "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/azure" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/gke" +) + +const ( + errGetPC = "cannot get ProviderConfig" + errGetCreds = "cannot get credentials" + errCreateRestConfig = "cannot create new REST config using provider secret" + errExtractGoogleCredentials = "cannot extract Google Application Credentials" + errInjectGoogleCredentials = "cannot wrap REST client with Google Application Credentials" + errExtractAzureCredentials = "failed to extract Azure Application Credentials" + errInjectAzureCredentials = "failed to wrap REST client with Azure Application Credentials" +) + +// ClientForProvider returns the client and *rest.config for the given provider +// config. +func ClientForProvider(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, *rest.Config, error) { //nolint:gocyclo + rc, err := configForProvider(ctx, inclusterClient, providerConfigName) + if err != nil { + return nil, nil, errors.Wrapf(err, "cannot get REST config for provider %q", providerConfigName) + } + k, err := client.New(rc, client.Options{}) + if err != nil { + return nil, nil, errors.Wrapf(err, "cannot create Kubernetes client for provider %q", providerConfigName) + } + return k, rc, nil +} + +// ConfigForProvider returns the *rest.config for the given provider config. +func configForProvider(ctx context.Context, local client.Client, providerConfigName string) (*rest.Config, error) { // nolint:gocyclo + pc := &v1alpha1.ProviderConfig{} + if err := local.Get(ctx, types.NamespacedName{Name: providerConfigName}, pc); err != nil { + return nil, errors.Wrap(err, errGetPC) + } + + var rc *rest.Config + var err error + + switch cd := pc.Spec.Credentials; cd.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + rc, err = rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, errCreateRestConfig) + } + default: + kc, err := resource.CommonCredentialExtractor(ctx, cd.Source, local, cd.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errGetCreds) + } + + ac, err := clientcmd.Load(kc) + if err != nil { + return nil, errors.Wrap(err, "failed to load kubeconfig") + } + + if rc, err = fromAPIConfig(ac); err != nil { + return nil, errors.Wrap(err, errCreateRestConfig) + } + } + + if id := pc.Spec.Identity; id != nil { + switch id.Type { + case v1alpha1.IdentityTypeGoogleApplicationCredentials: + switch id.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + if err := gke.WrapRESTConfig(ctx, rc, nil, gke.DefaultScopes...); err != nil { + return nil, errors.Wrap(err, errInjectGoogleCredentials) + } + default: + creds, err := resource.CommonCredentialExtractor(ctx, id.Source, local, id.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errExtractGoogleCredentials) + } + + if err := gke.WrapRESTConfig(ctx, rc, creds, gke.DefaultScopes...); err != nil { + return nil, errors.Wrap(err, errInjectGoogleCredentials) + } + } + case v1alpha1.IdentityTypeAzureServicePrincipalCredentials: + switch id.Source { //nolint:exhaustive + case xpv1.CredentialsSourceInjectedIdentity: + return nil, errors.Errorf("%s is not supported as identity source for identity type %s", + xpv1.CredentialsSourceInjectedIdentity, v1alpha1.IdentityTypeAzureServicePrincipalCredentials) + default: + creds, err := resource.CommonCredentialExtractor(ctx, id.Source, local, id.CommonCredentialSelectors) + if err != nil { + return nil, errors.Wrap(err, errExtractAzureCredentials) + } + + if err := azure.WrapRESTConfig(ctx, rc, creds); err != nil { + return nil, errors.Wrap(err, errInjectAzureCredentials) + } + } + default: + return nil, errors.Errorf("unknown identity type: %s", id.Type) + } + } + + return rc, nil +} + +func fromAPIConfig(c *api.Config) (*rest.Config, error) { + if c.CurrentContext == "" { + return nil, errors.New("currentContext not set in kubeconfig") + } + ctx := c.Contexts[c.CurrentContext] + cluster := c.Clusters[ctx.Cluster] + if cluster == nil { + return nil, errors.Errorf("cluster for currentContext (%s) not found", c.CurrentContext) + } + user := c.AuthInfos[ctx.AuthInfo] + if user == nil { + // We don't require a user because it's possible user + // authorization configuration will be loaded from a separate + // set of identity credentials (e.g. Google Application Creds). + user = &api.AuthInfo{} + } + config := &rest.Config{ + Host: cluster.Server, + Username: user.Username, + Password: user.Password, + BearerToken: user.Token, + BearerTokenFile: user.TokenFile, + Impersonate: rest.ImpersonationConfig{ + UserName: user.Impersonate, + Groups: user.ImpersonateGroups, + Extra: user.ImpersonateUserExtra, + }, + AuthProvider: user.AuthProvider, + ExecProvider: user.Exec, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: cluster.InsecureSkipTLSVerify, + ServerName: cluster.TLSServerName, + CertData: user.ClientCertificateData, + KeyData: user.ClientKeyData, + CAData: cluster.CertificateAuthorityData, + }, + } + + // NOTE(tnthornton): these values match the burst and QPS values in kubectl. + // xref: https://github.com/kubernetes/kubernetes/pull/105520 + config.Burst = 300 + config.QPS = 50 + + return config, nil +} diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 2d8d4e32..d42026c1 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -55,7 +55,7 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" apisv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/kube" "github.com/crossplane-contrib/provider-kubernetes/internal/features" ) @@ -137,7 +137,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit sanitizeSecrets: sanitizeSecrets, kube: mgr.GetClient(), usage: resource.NewProviderConfigUsageTracker(mgr.GetClient(), &apisv1alpha1.ProviderConfigUsage{}), - clientForProviderFn: clients.ClientForProvider, + clientForProviderFn: kube.ClientForProvider, } cb := ctrl.NewControllerManagedBy(mgr). @@ -196,7 +196,7 @@ type connector struct { kindObserver KindObserver - clientForProviderFn func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) + clientForProviderFn func(ctx context.Context, local client.Client, providerConfigName string) (client.Client, *rest.Config, error) } func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.ExternalClient, error) { //nolint:gocyclo @@ -212,15 +212,19 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E return nil, errors.Wrap(err, errTrackPCUsage) } - k, err := c.clientForProviderFn(ctx, c.kube, cr.GetProviderConfigReference().Name) + k, rc, err := c.clientForProviderFn(ctx, c.kube, cr.GetProviderConfigReference().Name) if err != nil { return nil, errors.Wrap(err, errNewKubernetesClient) } return &external{ - logger: c.logger, - client: k, + logger: c.logger, + client: resource.ClientApplicator{ + Client: k, + Applicator: resource.NewAPIPatchingApplicator(k), + }, + rest: rc, localClient: c.kube, sanitizeSecrets: c.sanitizeSecrets, @@ -230,8 +234,9 @@ func (c *connector) Connect(ctx context.Context, mg resource.Managed) (managed.E type external struct { logger logging.Logger - client clients.ClusterClient - // localClient is specifically used to connect to local cluster + client resource.ClientApplicator + rest *rest.Config + // localClient is specifically used to connect to local cluster, a.k.a control plane. localClient client.Client sanitizeSecrets bool @@ -259,7 +264,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex } if c.shouldWatch(cr) { - c.kindObserver.WatchResources(c.client.GetConfig(), cr.Spec.ProviderConfigReference.Name, desired.GroupVersionKind()) + c.kindObserver.WatchResources(c.rest, cr.Spec.ProviderConfigReference.Name, desired.GroupVersionKind()) } observed := desired.DeepCopy() diff --git a/internal/controller/object/object_test.go b/internal/controller/object/object_test.go index 814d685a..048deb4d 100644 --- a/internal/controller/object/object_test.go +++ b/internal/controller/object/object_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,7 +44,6 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" kubernetesv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients" ) const ( @@ -239,7 +239,7 @@ func Test_connector_Connect(t *testing.T) { type args struct { client client.Client - clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) + clientForProvider client.Client usage resource.Tracker mg resource.Managed } @@ -269,40 +269,24 @@ func Test_connector_Connect(t *testing.T) { }, "Success": { args: args{ - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { - return &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{}, - }, - }, nil - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), + clientForProvider: &test.MockClient{}, + usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), + mg: kubernetesObject(), }, want: want{ err: nil, }, }, - "ErrorGettingClientForProvider": { - args: args{ - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { - return nil, errBoom - }, - usage: resource.TrackerFn(func(ctx context.Context, mg resource.Managed) error { return nil }), - mg: kubernetesObject(), - }, - want: want{ - err: errors.Wrap(errBoom, errNewKubernetesClient), - }, - }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { c := &connector{ - logger: logging.NewNopLogger(), - kube: tc.args.client, - clientForProviderFn: tc.args.clientForProvider, - usage: tc.usage, + logger: logging.NewNopLogger(), + kube: tc.args.client, + clientForProviderFn: func(ctx context.Context, local client.Client, providerConfigName string) (client.Client, *rest.Config, error) { + return tc.args.clientForProvider, nil, nil + }, + usage: tc.usage, } _, gotErr := c.Connect(context.Background(), tc.args.mg) if diff := cmp.Diff(tc.want.err, gotErr, test.EquateErrors()); diff != "" { @@ -314,7 +298,7 @@ func Test_connector_Connect(t *testing.T) { func Test_helmExternal_Observe(t *testing.T) { type args struct { - client clients.ClusterClient + client resource.ClientApplicator mg resource.Managed } type want struct { @@ -336,11 +320,9 @@ func Test_helmExternal_Observe(t *testing.T) { "NoKubernetesObjectExists": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(kerrors.NewNotFound(schema.GroupResource{}, "")), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(kerrors.NewNotFound(schema.GroupResource{}, "")), }, }, }, @@ -362,11 +344,9 @@ func Test_helmExternal_Observe(t *testing.T) { "FailedToGet": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(errBoom), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(errBoom), }, }, }, @@ -377,14 +357,12 @@ func Test_helmExternal_Observe(t *testing.T) { "NoLastAppliedAnnotation": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *externalResource() - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *externalResource() + return nil + }), }, }, }, @@ -396,17 +374,15 @@ func Test_helmExternal_Observe(t *testing.T) { "NotUpToDate": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, - ) - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, + ) + return nil + }), }, }, }, @@ -422,17 +398,15 @@ func Test_helmExternal_Observe(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace"}`, - ) - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace"}`, + ) + return nil + }), }, }, }, @@ -448,14 +422,12 @@ func Test_helmExternal_Observe(t *testing.T) { "UpToDate": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + return nil + }), }, }, }, @@ -474,14 +446,12 @@ func Test_helmExternal_Observe(t *testing.T) { obj.Spec.References = objectReferences() obj.Spec.References[0].PatchesFrom.FieldPath = ptr.To("nonexistent_field") }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + }), }, }, }, @@ -496,11 +466,9 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = objectReferences() }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(errBoom), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(errBoom), }, }, }, @@ -516,20 +484,18 @@ func Test_helmExternal_Observe(t *testing.T) { obj.ObjectMeta.DeletionTimestamp = &metav1.Time{Time: time.Now()} obj.Spec.References = objectReferences() }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == testReferenceObjectName { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - } else if key.Name == externalResourceName { - return kerrors.NewNotFound(schema.GroupResource{}, "") - } - return errBoom - }, - MockUpdate: test.NewMockUpdateFn(nil), + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key.Name == testReferenceObjectName { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + } else if key.Name == externalResourceName { + return kerrors.NewNotFound(schema.GroupResource{}, "") + } + return errBoom }, + MockUpdate: test.NewMockUpdateFn(nil), }, }, }, @@ -543,21 +509,19 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = objectReferences() }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - if key.Name == testReferenceObjectName { - *obj.(*unstructured.Unstructured) = *referenceObject() - return nil - } else if key.Name == externalResourceName { - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - return nil - } - return errBoom - }, - MockUpdate: test.NewMockUpdateFn(nil), + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if key.Name == testReferenceObjectName { + *obj.(*unstructured.Unstructured) = *referenceObject() + return nil + } else if key.Name == externalResourceName { + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + return nil + } + return errBoom }, + MockUpdate: test.NewMockUpdateFn(nil), }, }, }, @@ -575,11 +539,9 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.References = []v1alpha2.Reference{{}} }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil), }, }, }, @@ -605,24 +567,22 @@ func Test_helmExternal_Observe(t *testing.T) { }, } }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - switch key.Name { - case externalResourceName: - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - case testSecretName: - *obj.(*unstructured.Unstructured) = unstructured.Unstructured{ - Object: map[string]interface{}{ - "data": map[string]interface{}{ - "db-password": "MTIzNDU=", - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + switch key.Name { + case externalResourceName: + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + case testSecretName: + *obj.(*unstructured.Unstructured) = unstructured.Unstructured{ + Object: map[string]interface{}{ + "data": map[string]interface{}{ + "db-password": "MTIzNDU=", }, - } + }, } - return nil - }, + } + return nil }, }, }, @@ -653,18 +613,16 @@ func Test_helmExternal_Observe(t *testing.T) { }, } }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { - switch key.Name { - case externalResourceName: - *obj.(*unstructured.Unstructured) = *upToDateExternalResource() - case testSecretName: - return errBoom - } - return nil - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { + switch key.Name { + case externalResourceName: + *obj.(*unstructured.Unstructured) = *upToDateExternalResource() + case testSecretName: + return errBoom + } + return nil }, }, }, @@ -678,17 +636,15 @@ func Test_helmExternal_Observe(t *testing.T) { mg: kubernetesObject(func(obj *v1alpha2.Object) { obj.Spec.ManagementPolicies = xpv1.ManagementPolicies{xpv1.ManagementActionObserve} }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { - *obj.(*unstructured.Unstructured) = - *externalResourceWithLastAppliedConfigAnnotation( - `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, - ) - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockGet: test.NewMockGetFn(nil, func(obj client.Object) error { + *obj.(*unstructured.Unstructured) = + *externalResourceWithLastAppliedConfigAnnotation( + `{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"crossplane-system", "labels": {"old-label":"gone"}}}`, + ) + return nil + }), }, }, }, @@ -719,7 +675,7 @@ func Test_helmExternal_Observe(t *testing.T) { func Test_helmExternal_Create(t *testing.T) { type args struct { - client clients.ClusterClient + client resource.ClientApplicator mg resource.Managed } type want struct { @@ -751,11 +707,9 @@ func Test_helmExternal_Create(t *testing.T) { "FailedToCreate": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(errBoom), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(errBoom), }, }, }, @@ -770,20 +724,18 @@ func Test_helmExternal_Create(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { - _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] - if !ok { - t.Errorf("Last applied annotation not set with create") - } - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { + _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] + if !ok { + t.Errorf("Last applied annotation not set with create") + } + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), }, }, }, @@ -794,17 +746,15 @@ func Test_helmExternal_Create(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { - _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] - if !ok { - t.Errorf("Last applied annotation not set with create") - } - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockCreate: test.NewMockCreateFn(nil, func(obj client.Object) error { + _, ok := obj.GetAnnotations()[corev1.LastAppliedConfigAnnotation] + if !ok { + t.Errorf("Last applied annotation not set with create") + } + return nil + }), }, }, }, @@ -833,7 +783,7 @@ func Test_helmExternal_Create(t *testing.T) { func Test_helmExternal_Update(t *testing.T) { type args struct { - client clients.ClusterClient + client resource.ClientApplicator mg resource.Managed } type want struct { @@ -865,12 +815,10 @@ func Test_helmExternal_Update(t *testing.T) { "FailedToApply": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { - return errBoom - }), - }, + client: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { + return errBoom + }), }, }, want: want{ @@ -884,15 +832,13 @@ func Test_helmExternal_Update(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(ctx context.Context, obj client.Object, op ...resource.ApplyOption) error { - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), - }, + client: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(ctx context.Context, obj client.Object, op ...resource.ApplyOption) error { + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), }, }, want: want{ @@ -902,12 +848,10 @@ func Test_helmExternal_Update(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { - return nil - }), - }, + client: resource.ClientApplicator{ + Applicator: resource.ApplyFn(func(context.Context, client.Object, ...resource.ApplyOption) error { + return nil + }), }, }, want: want{ @@ -935,7 +879,7 @@ func Test_helmExternal_Update(t *testing.T) { func Test_helmExternal_Delete(t *testing.T) { type args struct { - client clients.ClusterClient + client resource.ClientApplicator mg resource.Managed } type want struct { @@ -966,11 +910,9 @@ func Test_helmExternal_Delete(t *testing.T) { "FailedToDelete": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(errBoom), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(errBoom), }, }, }, @@ -985,16 +927,14 @@ func Test_helmExternal_Delete(t *testing.T) { "apiVersion": "v1", "kind": "Namespace" }`) }), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(nil, func(obj client.Object) error { - if obj.GetName() != testObjectName { - t.Errorf("Name should default to object name when not provider in manifest") - } - return nil - }), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(nil, func(obj client.Object) error { + if obj.GetName() != testObjectName { + t.Errorf("Name should default to object name when not provider in manifest") + } + return nil + }), }, }, }, @@ -1005,11 +945,9 @@ func Test_helmExternal_Delete(t *testing.T) { "Success": { args: args{ mg: kubernetesObject(), - client: &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: &test.MockClient{ - MockDelete: test.NewMockDeleteFn(nil), - }, + client: resource.ClientApplicator{ + Client: &test.MockClient{ + MockDelete: test.NewMockDeleteFn(nil), }, }, }, diff --git a/internal/controller/observedobjectcollection/reconciler.go b/internal/controller/observedobjectcollection/reconciler.go index 7620051c..825e0da7 100644 --- a/internal/controller/observedobjectcollection/reconciler.go +++ b/internal/controller/observedobjectcollection/reconciler.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -43,7 +44,7 @@ import ( "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients" + "github.com/crossplane-contrib/provider-kubernetes/internal/clients/kube" ) const ( @@ -59,7 +60,7 @@ type Reconciler struct { client client.Client log logging.Logger pollInterval func() time.Duration - clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) + clientForProvider func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, *rest.Config, error) observedObjectName func(collection client.Object, matchedObject client.Object) (string, error) } @@ -73,7 +74,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, pollJitter time.Duration) err pollInterval: func() time.Duration { return o.PollInterval + +time.Duration((rand.Float64()-0.5)*2*float64(pollJitter)) //nolint }, - clientForProvider: clients.ClientForProvider, + clientForProvider: kube.ClientForProvider, observedObjectName: observedObjectName, } @@ -124,7 +125,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct log.Info("Reconciling") // Get client for the referenced provider config. - clusterClient, err := r.clientForProvider(ctx, r.client, c.Spec.ProviderConfigReference.Name) + clusterClient, _, err := r.clientForProvider(ctx, r.client, c.Spec.ProviderConfigReference.Name) if err != nil { werr := errors.Wrap(err, errNewKubernetesClient) c.Status.SetConditions(xpv1.ReconcileError(werr)) diff --git a/internal/controller/observedobjectcollection/reconciler_test.go b/internal/controller/observedobjectcollection/reconciler_test.go index ad984cde..a3f898bf 100644 --- a/internal/controller/observedobjectcollection/reconciler_test.go +++ b/internal/controller/observedobjectcollection/reconciler_test.go @@ -32,18 +32,17 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" "github.com/crossplane/crossplane-runtime/pkg/logging" - "github.com/crossplane/crossplane-runtime/pkg/resource" "github.com/crossplane/crossplane-runtime/pkg/test" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" "github.com/crossplane-contrib/provider-kubernetes/apis/observedobjectcollection/v1alpha1" - "github.com/crossplane-contrib/provider-kubernetes/internal/clients" ) func TestReconciler(t *testing.T) { @@ -382,12 +381,8 @@ func TestReconciler(t *testing.T) { r := &Reconciler{ client: tc.args.client, log: logging.NewNopLogger(), - clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (clients.ClusterClient, error) { - return &clients.ApplicatorClientWithConfig{ - ClientApplicator: resource.ClientApplicator{ - Client: tc.args.client, - }, - }, nil + clientForProvider: func(ctx context.Context, inclusterClient client.Client, providerConfigName string) (client.Client, *rest.Config, error) { + return tc.args.client, nil, nil }, observedObjectName: func(collection client.Object, matchedObject client.Object) (string, error) { return fmt.Sprintf("%s-%s", collection.GetName(), matchedObject.GetName()), nil From 5723426757f2c7c9b836ce1383fa9d4a3e23c563 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 10 May 2024 15:08:05 +0300 Subject: [PATCH 32/36] Rely on garbage collect and override default DefaultWatchErrorHandler Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- apis/object/v1alpha1/types.go | 4 +- apis/object/v1alpha2/types.go | 4 +- internal/controller/object/informers.go | 79 ++++++++----------- internal/controller/object/object.go | 27 +------ .../kubernetes.crossplane.io_objects.yaml | 8 +- 5 files changed, 40 insertions(+), 82 deletions(-) diff --git a/apis/object/v1alpha1/types.go b/apis/object/v1alpha1/types.go index dfed2914..beaff8f0 100644 --- a/apis/object/v1alpha1/types.go +++ b/apis/object/v1alpha1/types.go @@ -127,8 +127,8 @@ type ObjectSpec struct { // Watch enables watching the referenced or managed kubernetes resources. // // THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored - // unless the relevant Crossplane feature flag is enabled, and may be - // changed or removed without notice. + // unless "watches" feature gate is enabled, and may be changed or removed + // without notice. // +optional // +kubebuilder:default=false Watch bool `json:"watch,omitempty"` diff --git a/apis/object/v1alpha2/types.go b/apis/object/v1alpha2/types.go index 426def82..337212c0 100644 --- a/apis/object/v1alpha2/types.go +++ b/apis/object/v1alpha2/types.go @@ -100,8 +100,8 @@ type ObjectSpec struct { // Watch enables watching the referenced or managed kubernetes resources. // // THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored - // unless the relevant Crossplane feature flag is enabled, and may be - // changed or removed without notice. + // unless "watches" feature gate is enabled, and may be changed or removed + // without notice. // +optional // +kubebuilder:default=false Watch bool `json:"watch,omitempty"` diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index b38fe235..6a43cc67 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -18,6 +18,7 @@ package object import ( "context" + "io" "strings" "sync" @@ -103,10 +104,10 @@ func (i *resourceInformers) Start(ctx context.Context, h handler.EventHandler, q // every reconcile to make resourceInformers aware of the referenced or managed // resources of the given Object. // -// Note that this complements garbageCollectResourceInformers which regularly +// Note that this complements cleanupResourceInformers which regularly // garbage collects resource informers that are no longer referenced by // any Object. -func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { +func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) { // nolint:gocyclo // we need to handle all cases. if rc == nil { rc = i.config } @@ -122,7 +123,15 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin log := i.log.WithValues("providerConfig", providerConfig, "gvk", gvk.String()) - ca, err := cache.New(rc, cache.Options{}) + ca, err := cache.New(rc, cache.Options{ + DefaultWatchErrorHandler: func(r *kcache.Reflector, err error) { + if errors.Is(io.EOF, err) { + // Watch closed normally. + return + } + log.Debug("Watch error - probably remote cluster api is gone", "error", err) + }, + }) if err != nil { log.Debug("failed creating a cache", "error", err) continue @@ -163,6 +172,9 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin i.sink(providerConfig, ev) }, DeleteFunc: func(obj interface{}) { + if final, ok := obj.(kcache.DeletedFinalStateUnknown); ok { + obj = final.Obj + } ev := runtimeevent.GenericEvent{ Object: obj.(client.Object), } @@ -206,57 +218,26 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin } } -func (i *resourceInformers) StopWatchingResources(ctx context.Context, providerConfig string, gvks ...schema.GroupVersionKind) { - i.lock.Lock() - defer i.lock.Unlock() - - for _, gvk := range gvks { - ca, found := i.resourceCaches[gvkWithConfig{providerConfig: providerConfig, gvk: gvk}] - if !found { - continue - } - // Check if there are any other objects referencing this resource GVK. - list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(providerConfig, gvk.Kind, gvk.Group, gvk.Version)) - continue - } - - inUse := false - for _, o := range list.Items { - // We only care about objects that are not being deleted. Otherwise, - // we are getting into deadlocks while stopping the watches during - // deletion. - if o.GetDeletionTimestamp().IsZero() { - inUse = true - break - } - } - if inUse { - continue - } - - ca.cancelFn() - i.log.Info("Stopped resource watch", "provider config", providerConfig, "gvk", gvk) - delete(i.resourceCaches, gvkWithConfig{providerConfig: providerConfig, gvk: gvk}) - } -} - -// garbageCollectResourceInformers garbage collects resource informers that are +// cleanupResourceInformers garbage collects resource informers that are // no longer referenced by any Object. Ideally, all resource informers should // stopped/cleaned up when the Object is deleted. However, in practice, this // is not always the case. This method is a safety net to clean up resource // informers that are no longer referenced by any Object. -func (i *resourceInformers) garbageCollectResourceInformers(ctx context.Context) { - i.lock.Lock() - defer i.lock.Unlock() +func (i *resourceInformers) cleanupResourceInformers(ctx context.Context) { + // copy map to avoid locking it for the entire duration of the loop + i.lock.RLock() + resourceCaches := make(map[gvkWithConfig]resourceCache, len(i.resourceCaches)) + for gc, ca := range i.resourceCaches { + resourceCaches[gc] = ca + } + i.lock.RUnlock() // stop old informers i.log.Debug("Running garbage collection for resource informers", "count", len(i.resourceCaches)) - for gh, ca := range i.resourceCaches { + for gc, ca := range resourceCaches { list := v1alpha2.ObjectList{} - if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)}); err != nil { - i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(gh.providerConfig, gh.gvk.Kind, gh.gvk.Group, gh.gvk.Version)) + if err := i.objectsCache.List(ctx, &list, client.MatchingFields{resourceRefGVKsIndex: refKeyProviderGVK(gc.providerConfig, gc.gvk.Kind, gc.gvk.Group, gc.gvk.Version)}); err != nil { + i.log.Debug("cannot list objects referencing a certain resource GVK", "error", err, "fieldSelector", resourceRefGVKsIndex+"="+refKeyProviderGVK(gc.providerConfig, gc.gvk.Kind, gc.gvk.Group, gc.gvk.Version)) continue } @@ -265,8 +246,10 @@ func (i *resourceInformers) garbageCollectResourceInformers(ctx context.Context) } ca.cancelFn() - i.log.Info("Stopped resource watch", "provider config", gh.providerConfig, "gvk", gh.gvk) - delete(i.resourceCaches, gh) + i.log.Info("Stopped resource watch", "provider config", gc.providerConfig, "gvk", gc.gvk) + i.lock.Lock() + delete(i.resourceCaches, gc) + i.lock.Unlock() } } diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index d42026c1..072be877 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -102,10 +102,6 @@ type KindObserver interface { // WatchResources starts a watch of the given kinds to trigger reconciles // when a referenced or managed objects of those kinds changes. WatchResources(rc *rest.Config, providerConfig string, gvks ...schema.GroupVersionKind) - - // UnwatchResources stops watching the given kinds if they are no longer - // referenced or managed by any other Object that is not being deleted. - StopWatchingResources(ctx context.Context, providerConfig string, gvks ...schema.GroupVersionKind) } // Setup adds a controller that reconciles Object managed resources. @@ -164,7 +160,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit conn.kindObserver = &i if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - wait.UntilWithContext(ctx, i.garbageCollectResourceInformers, time.Minute) + wait.UntilWithContext(ctx, i.cleanupResourceInformers, time.Minute) return nil })); err != nil { return errors.Wrap(err, "cannot add cleanup referenced resource informers runnable") @@ -354,27 +350,6 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error { return err } - if c.shouldWatch(cr) { - c.kindObserver.StopWatchingResources(ctx, cr.Spec.ProviderConfigReference.Name, obj.GroupVersionKind()) - if len(cr.Spec.References) > 0 { - gvks := make([]schema.GroupVersionKind, 0, len(cr.Spec.References)) - for _, ref := range cr.Spec.References { - if ref.DependsOn == nil && ref.PatchesFrom == nil { - continue - } - - refAPIVersion, refKind, _, _ := getReferenceInfo(ref) - g, v := parseAPIVersion(refAPIVersion) - gvks = append(gvks, schema.GroupVersionKind{ - Group: g, - Version: v, - Kind: refKind, - }) - } - c.kindObserver.StopWatchingResources(ctx, "", gvks...) - } - } - return errors.Wrap(resource.IgnoreNotFound(c.client.Delete(ctx, obj)), errDeleteObject) } diff --git a/package/crds/kubernetes.crossplane.io_objects.yaml b/package/crds/kubernetes.crossplane.io_objects.yaml index f3aded13..13ee7dea 100644 --- a/package/crds/kubernetes.crossplane.io_objects.yaml +++ b/package/crds/kubernetes.crossplane.io_objects.yaml @@ -407,8 +407,8 @@ spec: THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored - unless the relevant Crossplane feature flag is enabled, and may be - changed or removed without notice. + unless "watches" feature gate is enabled, and may be changed or removed + without notice. type: boolean writeConnectionSecretToRef: description: |- @@ -851,8 +851,8 @@ spec: THIS IS AN ALPHA FIELD. Do not use it in production. It is not honored - unless the relevant Crossplane feature flag is enabled, and may be - changed or removed without notice. + unless "watches" feature gate is enabled, and may be changed or removed + without notice. type: boolean writeConnectionSecretToRef: description: |- From d1842e49d46232b89906ca849bcdbd246be1cad9 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 10 May 2024 16:22:06 +0300 Subject: [PATCH 33/36] Add e2e for watching object Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- Makefile | 2 +- build | 2 +- examples/object/object-watching.yaml | 40 ++++++++++++++ examples/object/object.yaml | 2 + .../object/testhooks/validate-watching.sh | 53 +++++++++++++++++++ 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 examples/object/object-watching.yaml create mode 100755 examples/object/testhooks/validate-watching.sh diff --git a/Makefile b/Makefile index 4bc23a4f..7abe9c54 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ CROSSPLANE_NAMESPACE = crossplane-system -include build/makelib/local.xpkg.mk -include build/makelib/controlplane.mk -UPTEST_EXAMPLE_LIST ?= "examples/object/object.yaml" +UPTEST_EXAMPLE_LIST ?= "examples/object/object.yaml,examples/object/object-watching.yaml" uptest: $(UPTEST) $(KUBECTL) $(KUTTL) @$(INFO) running automated tests @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE} $(UPTEST) e2e "$(UPTEST_EXAMPLE_LIST)" --setup-script=cluster/test/setup.sh || $(FAIL) diff --git a/build b/build index cdee0068..93d68794 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit cdee00685cfc90fce40d3f6a5eb9c8f044edbaa3 +Subproject commit 93d68794581d7991b0d6816a077e0ab60eb0eb24 diff --git a/examples/object/object-watching.yaml b/examples/object/object-watching.yaml new file mode 100644 index 00000000..cfd9e012 --- /dev/null +++ b/examples/object/object-watching.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: kubernetes.crossplane.io/v1alpha2 +kind: Object +metadata: + name: foo + annotations: + uptest.upbound.io/post-assert-hook: testhooks/validate-watching.sh + uptest.upbound.io/timeout: "60" +spec: + # Watch for changes to the Namespace object. + # Watching resources is an alpha feature and needs to be enabled with --enable-watches + # in the provider to get this configuration working. + watch: true + references: + # Use patchesFrom to patch field from other k8s resource to this object + - patchesFrom: + apiVersion: v1 + kind: Secret + name: bar + namespace: default + fieldPath: data.key + toFieldPath: data.key-from-bar + forProvider: + manifest: + apiVersion: v1 + kind: Secret + metadata: + namespace: default + stringData: + another-key: another-value + providerConfigRef: + name: kubernetes-provider +--- +apiVersion: v1 +kind: Secret +metadata: + name: bar + namespace: default +stringData: + key: some-value diff --git a/examples/object/object.yaml b/examples/object/object.yaml index b028dd23..749b8c3f 100644 --- a/examples/object/object.yaml +++ b/examples/object/object.yaml @@ -2,6 +2,8 @@ apiVersion: kubernetes.crossplane.io/v1alpha2 kind: Object metadata: name: sample-namespace + annotations: + uptest.upbound.io/timeout: "60" spec: # Watch for changes to the Namespace object. # Watching resources is an alpha feature and needs to be enabled with --enable-watches diff --git a/examples/object/testhooks/validate-watching.sh b/examples/object/testhooks/validate-watching.sh new file mode 100755 index 00000000..29a7bb8d --- /dev/null +++ b/examples/object/testhooks/validate-watching.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -aeuo pipefail + +# This script is used to validate the watch feature of the object, triggered by the +# uptest framework via `uptest.upbound.io/post-assert-hook`: https://github.com/crossplane/uptest/tree/e64457e2cce153ada54da686c8bf96143f3f6329?tab=readme-ov-file#hooks +KUBECTL="kubectl" + +VALUE=$(${KUBECTL} get secret bar -o jsonpath='{.data.key}' | base64 -d) +if [ "${VALUE}" == "new-value" ]; then + echo "This test has to pass in the first run since we're validating the realtime watching behaviour." + exit 1 +fi + +echo "Enabling watch feature for the provider" +${KUBECTL} patch deploymentruntimeconfig runtimeconfig-provider-kubernetes --type='json' -p='[{"op":"replace","path":"/spec/deploymentTemplate/spec/template/spec/containers/0/args", "value":["--debug", "--enable-watches"]}]' + +sleep 3 + +echo "Patching referenced secret" +${KUBECTL} patch secret bar --type='merge' -p='{"stringData":{"key":"new-value"}}' + +sleep 3 + +echo "Checking if the managed secret has been updated" +VALUE=$(${KUBECTL} get secret foo -o jsonpath='{.data.key-from-bar}' | base64 -d) +if [ "${VALUE}" != "new-value" ]; then + echo "Expected value to be 'new-value' but got '${VALUE}'" + exit 1 +fi +echo "Checking if the managed secret has been updated...Success" + +echo "Patching managed secret" +${KUBECTL} patch secret foo --type='merge' -p='{"stringData":{"a-new-key":"with-new-value"}}' + +sleep 3 + +echo "Checking if the object grabbed the new value at status.atProvider" +VALUE=$(${KUBECTL} get object foo -o jsonpath='{.status.atProvider.manifest.data.a-new-key}' | base64 -d) + +if [ "${VALUE}" != "with-new-value" ]; then + echo "Expected value to be 'with-new-value' but got '${VALUE}'" + exit 1 +fi +echo "Checking if the object grabbed the new value at status.atProvider...Success" + +# TODO(turkenh): Add one more test case to validate the drift is reverted back to the desired state +# in realtime after https://github.com/crossplane-contrib/provider-kubernetes/issues/37 is resolved. + +echo "Successfully validated the watch feature!" + +echo "Disabling watch feature for the provider" +${KUBECTL} patch deploymentruntimeconfig runtimeconfig-provider-kubernetes --type='json' -p='[{"op":"replace","path":"/spec/deploymentTemplate/spec/template/spec/containers/0/args", "value":["--debug"]}]' + From 683fa86a7252ad10bd400e7c059a6b0abf1ae499 Mon Sep 17 00:00:00 2001 From: Hasan Turken Date: Fri, 10 May 2024 18:27:14 +0300 Subject: [PATCH 34/36] Always pass update event to reconciler Signed-off-by: Hasan Turken Signed-off-by: Alexander Brandstedt --- internal/controller/object/informers.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/controller/object/informers.go b/internal/controller/object/informers.go index 6a43cc67..0daa8fc2 100644 --- a/internal/controller/object/informers.go +++ b/internal/controller/object/informers.go @@ -159,14 +159,8 @@ func (i *resourceInformers) WatchResources(rc *rest.Config, providerConfig strin i.sink(providerConfig, ev) }, UpdateFunc: func(oldObj, newObj interface{}) { - old := oldObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. - obj := newObj.(client.Object) //nolint:forcetypeassert // Will always be client.Object. - if old.GetResourceVersion() == obj.GetResourceVersion() { - return - } - ev := runtimeevent.GenericEvent{ - Object: obj, + Object: newObj.(client.Object), } i.sink(providerConfig, ev) From cdf816fb9041652390da5671af87837281b513b3 Mon Sep 17 00:00:00 2001 From: ezgidemirel Date: Tue, 16 Apr 2024 20:14:19 +0300 Subject: [PATCH 35/36] Introduce MR metrics Signed-off-by: ezgidemirel Signed-off-by: Alexander Brandstedt --- cmd/provider/main.go | 33 +++++++++---- go.mod | 23 +++++----- go.sum | 46 ++++++++++--------- internal/controller/object/object.go | 7 +++ .../kubernetes.crossplane.io_objects.yaml | 28 +++++++++++ ...ossplane.io_observedobjectcollections.yaml | 14 ++++++ ...ernetes.crossplane.io_providerconfigs.yaml | 7 +++ 7 files changed, 117 insertions(+), 41 deletions(-) diff --git a/cmd/provider/main.go b/cmd/provider/main.go index fca9670b..8c337ef2 100644 --- a/cmd/provider/main.go +++ b/cmd/provider/main.go @@ -28,12 +28,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/crossplane/crossplane-runtime/pkg/controller" "github.com/crossplane/crossplane-runtime/pkg/feature" "github.com/crossplane/crossplane-runtime/pkg/logging" "github.com/crossplane/crossplane-runtime/pkg/ratelimiter" + "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" + "github.com/crossplane/crossplane-runtime/pkg/statemetrics" "github.com/crossplane-contrib/provider-kubernetes/apis" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha1" @@ -51,14 +54,15 @@ const ( func main() { var ( - app = kingpin.New(filepath.Base(os.Args[0]), "Template support for Crossplane.").DefaultEnvars() - debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() - syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() - pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration() - pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() - leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() - maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("100").Int() - sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() + app = kingpin.New(filepath.Base(os.Args[0]), "Template support for Crossplane.").DefaultEnvars() + debug = app.Flag("debug", "Run with debug logging.").Short('d').Bool() + syncInterval = app.Flag("sync", "Controller manager sync period such as 300ms, 1.5h, or 2h45m").Short('s').Default("1h").Duration() + pollInterval = app.Flag("poll", "Poll interval controls how often an individual resource should be checked for drift.").Default("10m").Duration() + pollStateMetricInterval = app.Flag("poll-state-metric", "State metric recording interval").Default("5s").Duration() + pollJitterPercentage = app.Flag("poll-jitter-percentage", "Percentage of jitter to apply to poll interval. It cannot be negative, and must be less than 100.").Default("10").Uint() + leaderElection = app.Flag("leader-election", "Use leader election for the controller manager.").Short('l').Default("false").Envar("LEADER_ELECTION").Bool() + maxReconcileRate = app.Flag("max-reconcile-rate", "The number of concurrent reconciliations that may be running at one time.").Default("100").Int() + sanitizeSecrets = app.Flag("sanitize-secrets", "when enabled, redacts Secret data from Object status").Default("false").Envar("SANITIZE_SECRETS").Bool() enableManagementPolicies = app.Flag("enable-management-policies", "Enable support for Management Policies.").Default("true").Envar("ENABLE_MANAGEMENT_POLICIES").Bool() enableWatches = app.Flag("enable-watches", "Enable support for watching resources.").Default("false").Envar("ENABLE_WATCHES").Bool() @@ -124,6 +128,18 @@ func main() { }) kingpin.FatalIfError(err, "Cannot create controller manager") + mm := managed.NewMRMetricRecorder() + sm := statemetrics.NewMRStateMetrics() + + metrics.Registry.MustRegister(mm) + metrics.Registry.MustRegister(sm) + + mo := controller.MetricOptions{ + PollStateMetricInterval: *pollStateMetricInterval, + MRMetrics: mm, + MRStateMetrics: sm, + } + kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add Template APIs to scheme") o := controller.Options{ Logger: log, @@ -131,6 +147,7 @@ func main() { PollInterval: *pollInterval, GlobalRateLimiter: ratelimiter.NewGlobal(*maxReconcileRate), Features: &feature.Flags{}, + MetricOptions: &mo, } if *enableManagementPolicies { diff --git a/go.mod b/go.mod index aef6678d..8ecbe235 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/Azure/kubelogin v0.0.0-00010101000000-000000000000 - github.com/crossplane/crossplane-runtime v1.15.0-rc.1 + github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.4.0 @@ -13,9 +13,9 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/oauth2 v0.16.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 - k8s.io/api v0.29.1 - k8s.io/apimachinery v0.29.1 - k8s.io/client-go v0.29.1 + k8s.io/api v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/controller-tools v0.14.0 @@ -38,6 +38,7 @@ require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dave/jennifer v1.4.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -56,7 +57,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -79,24 +80,24 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cobra v1.8.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.18.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/retry.v1 v1.0.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.1 // indirect - k8s.io/component-base v0.29.1 // indirect + k8s.io/component-base v0.29.3 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 635bce6e..7f772042 100644 --- a/go.sum +++ b/go.sum @@ -34,12 +34,14 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crossplane/crossplane-runtime v1.15.0-rc.1 h1:LkYZk/mRLZiJrL9gx6mM8t0Mvloy0nkfwy/BKmpC2wA= -github.com/crossplane/crossplane-runtime v1.15.0-rc.1/go.mod h1:kRcJjJQmBFrR2n/KhwL8wYS7xNfq3D8eK4JliEScOHI= +github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60 h1:subdCU8vHUDkaQAYBKKddCT2HpEK4A9fFEJ3nhGoTBc= +github.com/crossplane/crossplane-runtime v1.17.0-rc.0.0.20240509182037-b31be7747c60/go.mod h1:Pz2tdGVMF6KDGzHZOkvKro0nKc8EzK0sb/nSA7pH4Dc= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 h1:HigXs5tEQxWz0fcj8hzbU2UAZgEM7wPe0XRFOsrtF8Y= github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79/go.mod h1:+e4OaFlOcmr0JvINHl/yvEYBrZawzTgj6pQumOH1SS0= github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= @@ -89,8 +91,8 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -202,8 +204,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -219,8 +221,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -241,13 +243,13 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -275,8 +277,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -297,16 +299,16 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= -k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= -k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= -k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= -k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= -k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= -k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 072be877..8f318eaa 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -52,6 +52,7 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/ratelimiter" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/crossplane/crossplane-runtime/pkg/statemetrics" "github.com/crossplane-contrib/provider-kubernetes/apis/object/v1alpha2" apisv1alpha1 "github.com/crossplane-contrib/provider-kubernetes/apis/v1alpha1" @@ -126,6 +127,7 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit managed.WithLogger(l), managed.WithRecorder(event.NewAPIRecorder(mgr.GetEventRecorderFor(name))), managed.WithConnectionPublishers(cps...), + managed.WithMetricRecorder(o.MetricOptions.MRMetrics), } conn := &connector{ @@ -178,6 +180,11 @@ func Setup(mgr ctrl.Manager, o controller.Options, sanitizeSecrets bool, pollJit reconcilerOptions = append(reconcilerOptions, managed.WithManagementPolicies()) } + if err := mgr.Add(statemetrics.NewMRStateRecorder( + mgr.GetClient(), o.Logger, o.MetricOptions.MRStateMetrics, &v1alpha2.ObjectList{}, o.MetricOptions.PollStateMetricInterval)); err != nil { + return err + } + return cb.Complete(ratelimiter.NewReconciler(name, managed.NewReconciler(mgr, resource.ManagedKind(v1alpha2.ObjectGroupVersionKind), reconcilerOptions..., diff --git a/package/crds/kubernetes.crossplane.io_objects.yaml b/package/crds/kubernetes.crossplane.io_objects.yaml index 13ee7dea..06ece049 100644 --- a/package/crds/kubernetes.crossplane.io_objects.yaml +++ b/package/crds/kubernetes.crossplane.io_objects.yaml @@ -462,6 +462,13 @@ spec: A Message containing details about this condition's last transition from one status to another, if any. type: string + observedGeneration: + description: |- + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + type: integer reason: description: A Reason for this condition's last transition from one status to another. @@ -485,6 +492,13 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + observedGeneration: + description: |- + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + format: int64 + type: integer type: object required: - spec @@ -906,6 +920,13 @@ spec: A Message containing details about this condition's last transition from one status to another, if any. type: string + observedGeneration: + description: |- + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + type: integer reason: description: A Reason for this condition's last transition from one status to another. @@ -929,6 +950,13 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + observedGeneration: + description: |- + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + format: int64 + type: integer type: object required: - spec diff --git a/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml b/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml index 59384dbe..33551402 100644 --- a/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml +++ b/package/crds/kubernetes.crossplane.io_observedobjectcollections.yaml @@ -214,6 +214,13 @@ spec: A Message containing details about this condition's last transition from one status to another, if any. type: string + observedGeneration: + description: |- + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + type: integer reason: description: A Reason for this condition's last transition from one status to another. @@ -244,6 +251,13 @@ spec: MembershipLabel is the label set on each member of this collection and can be used for fetching them. type: object + observedGeneration: + description: |- + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + format: int64 + type: integer type: object required: - spec diff --git a/package/crds/kubernetes.crossplane.io_providerconfigs.yaml b/package/crds/kubernetes.crossplane.io_providerconfigs.yaml index ebd9df60..ba141a74 100644 --- a/package/crds/kubernetes.crossplane.io_providerconfigs.yaml +++ b/package/crds/kubernetes.crossplane.io_providerconfigs.yaml @@ -197,6 +197,13 @@ spec: A Message containing details about this condition's last transition from one status to another, if any. type: string + observedGeneration: + description: |- + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + type: integer reason: description: A Reason for this condition's last transition from one status to another. From 61b5cbc416ba674ad159945ab736ef1c650e6aea Mon Sep 17 00:00:00 2001 From: Alexander Brandstedt Date: Thu, 23 May 2024 11:36:40 +0200 Subject: [PATCH 36/36] change resolve logic to exclude secret when initial resolve runs in observer loop. This to stop secrets payload from ending in the MR and instead resolve one moretime for the secret just before use in Create/Update/Delete funcs Signed-off-by: Alexander Brandstedt --- cluster/kustomize/webhook/webhook.patch.yaml | 2 + internal/controller/object/object.go | 108 ++++++++++++++---- .../observedobjectcollection/reconciler.go | 2 +- .../kubernetes.crossplane.io_objects.yaml | 2 + 4 files changed, 90 insertions(+), 24 deletions(-) diff --git a/cluster/kustomize/webhook/webhook.patch.yaml b/cluster/kustomize/webhook/webhook.patch.yaml index b7d321ec..ca312d7f 100644 --- a/cluster/kustomize/webhook/webhook.patch.yaml +++ b/cluster/kustomize/webhook/webhook.patch.yaml @@ -10,4 +10,6 @@ spec: - v1 clientConfig: service: + name: provider-kubernetes + namespace: crossplane-system path: /convert \ No newline at end of file diff --git a/internal/controller/object/object.go b/internal/controller/object/object.go index 8f318eaa..2e9897ab 100644 --- a/internal/controller/object/object.go +++ b/internal/controller/object/object.go @@ -95,6 +95,8 @@ const ( errGetValueAtFieldPath = "cannot get value at fieldPath" errDecodeSecretData = "cannot decode secret data" errSanitizeSecretData = "cannot sanitize secret data" + + secretKind = "Secret" ) // KindObserver tracks kinds of referenced composed resources in order to start @@ -256,7 +258,7 @@ func (c *external) Observe(ctx context.Context, mg resource.Managed) (managed.Ex if !meta.WasDeleted(cr) { // If the object is not being deleted, we need to resolve references - if err := c.resolveReferencies(ctx, cr); err != nil { + if err := c.resolveReferencesOnObserverLookup(ctx, cr); err != nil { return managed.ExternalObservation{}, errors.Wrap(err, errResolveResourceReferences) } } @@ -303,6 +305,11 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext } c.logger.Debug("Creating", "resource", cr) + if cr.Kind == secretKind { + if err := c.resolveReferencesSecret(ctx, cr); err != nil { + return managed.ExternalCreation{}, errors.Wrap(err, errResolveResourceReferences) + } + } obj, err := getDesired(cr) if err != nil { @@ -328,6 +335,11 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext c.logger.Debug("Updating", "resource", cr) + if cr.Kind == secretKind { + if err := c.resolveReferencesSecret(ctx, cr); err != nil { + return managed.ExternalUpdate{}, errors.Wrap(err, errResolveResourceReferences) + } + } obj, err := getDesired(cr) if err != nil { return managed.ExternalUpdate{}, err @@ -352,6 +364,11 @@ func (c *external) Delete(ctx context.Context, mg resource.Managed) error { c.logger.Debug("Deleting", "resource", cr) + if cr.Kind == secretKind { + if err := c.resolveReferencesSecret(ctx, cr); err != nil { + return errors.Wrap(err, errResolveResourceReferences) + } + } obj, err := getDesired(cr) if err != nil { return err @@ -395,7 +412,7 @@ func (c *external) setObserved(obj *v1alpha2.Object, observed *unstructured.Unst var err error if c.sanitizeSecrets { - if observed.GetKind() == "Secret" && observed.GetAPIVersion() == "v1" { + if observed.GetKind() == secretKind && observed.GetAPIVersion() == "v1" { data := map[string][]byte{"redacted": []byte(nil)} if err = fieldpath.Pave(observed.Object).SetValue("data", data); err != nil { return errors.Wrap(err, errSanitizeSecretData) @@ -479,11 +496,35 @@ func getReferenceInfo(ref v1alpha2.Reference) (string, string, string, string) { return apiVersion, kind, namespace, name } -// resolveReferencies resolves references for the current Object. If it fails to +func (c *external) resolveReferencesAndPatch(ctx context.Context, obj *v1alpha2.Object, ref v1alpha2.Reference) (string, string, bool, error) { + refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) + res := &unstructured.Unstructured{} + res.SetAPIVersion(refAPIVersion) + res.SetKind(refKind) + // Try to get referenced resource + err := c.localClient.Get(ctx, client.ObjectKey{ + Namespace: refNamespace, + Name: refName, + }, res) + + if err != nil { + return "", "", true, errors.Wrap(err, errGetReferencedResource) + } + + // Patch fields if any + if ref.PatchesFrom != nil && ref.PatchesFrom.FieldPath != nil { + if err := ref.ApplyFromFieldPathPatch(res, obj); err != nil { + return "", "", true, errors.Wrap(err, errPatchFromReferencedResource) + } + } + return refAPIVersion, refKind, false, nil +} + +// resolveReferencesOnObserverLookup resolves references for the current Object. If it fails to // resolve some reference, e.g.: due to reference not ready, it will then return // error and requeue to wait for resolving it next time. -func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) error { - c.logger.Debug("Resolving referencies.") +func (c *external) resolveReferencesOnObserverLookup(ctx context.Context, obj *v1alpha2.Object) error { + c.logger.Debug("Resolving references.") // Loop through references to resolve each referenced resource gvks := make([]schema.GroupVersionKind, 0, len(obj.Spec.References)) @@ -492,25 +533,15 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) continue } - refAPIVersion, refKind, refNamespace, refName := getReferenceInfo(ref) - res := &unstructured.Unstructured{} - res.SetAPIVersion(refAPIVersion) - res.SetKind(refKind) - // Try to get referenced resource - err := c.localClient.Get(ctx, client.ObjectKey{ - Namespace: refNamespace, - Name: refName, - }, res) - - if err != nil { - return errors.Wrap(err, errGetReferencedResource) + // skip of secret type to not leak in to MR. secret lookup is done in Create/Update/Delete functions + // applied by resolveReferencesSecret from the functions. + if ref.PatchesFrom != nil && ref.PatchesFrom.DependsOn.Kind == secretKind { + continue } - // Patch fields if any - if ref.PatchesFrom != nil && ref.PatchesFrom.FieldPath != nil { - if err := ref.ApplyFromFieldPathPatch(res, obj); err != nil { - return errors.Wrap(err, errPatchFromReferencedResource) - } + refAPIVersion, refKind, done, err := c.resolveReferencesAndPatch(ctx, obj, ref) + if done { + return err } g, v := parseAPIVersion(refAPIVersion) @@ -531,6 +562,37 @@ func (c *external) resolveReferencies(ctx context.Context, obj *v1alpha2.Object) return nil } +// resolveReferencesSecret more or less duplicate of resolveReferencesOnObserverLookup, but it will handle the secret +func (c *external) resolveReferencesSecret(ctx context.Context, obj *v1alpha2.Object) error { + c.logger.Debug("Resolving references.") + + // Loop through references to resolve each referenced resource + gvks := make([]schema.GroupVersionKind, 0, len(obj.Spec.References)) + for _, ref := range obj.Spec.References { + if ref.DependsOn == nil && ref.PatchesFrom == nil { + continue + } + + refAPIVersion, refKind, done, err := c.resolveReferencesAndPatch(ctx, obj, ref) + if done { + return err + } + + g, v := parseAPIVersion(refAPIVersion) + gvks = append(gvks, schema.GroupVersionKind{ + Group: g, + Version: v, + Kind: refKind, + }) + } + + if c.shouldWatch(obj) { + c.kindObserver.WatchResources(nil, "", gvks...) + } + + return nil +} + func (c *external) handleLastApplied(ctx context.Context, obj *v1alpha2.Object, last, desired *unstructured.Unstructured) (managed.ExternalObservation, error) { isUpToDate := false @@ -689,7 +751,7 @@ func connectionDetails(ctx context.Context, kube client.Client, connDetails []v1 s := fmt.Sprintf("%v", v) fv := []byte(s) // prevent secret data being encoded twice - if cd.Kind == "Secret" && cd.APIVersion == "v1" && strings.HasPrefix(cd.FieldPath, "data") { + if cd.Kind == secretKind && cd.APIVersion == "v1" && strings.HasPrefix(cd.FieldPath, "data") { fv, err = base64.StdEncoding.DecodeString(s) if err != nil { return mcd, errors.Wrap(err, errDecodeSecretData) diff --git a/internal/controller/observedobjectcollection/reconciler.go b/internal/controller/observedobjectcollection/reconciler.go index 825e0da7..b5e61bb9 100644 --- a/internal/controller/observedobjectcollection/reconciler.go +++ b/internal/controller/observedobjectcollection/reconciler.go @@ -259,7 +259,7 @@ func observedObjectPatch(name string, matchedObject unstructured.Unstructured, c }, ForProvider: v1alpha2.ObjectParameters{ Manifest: runtime.RawExtension{ - Raw: []byte(manifest), + Raw: []byte(manifest), // here the manifest is rendered and if a secret it will contain sensitive data ? }, }, }, diff --git a/package/crds/kubernetes.crossplane.io_objects.yaml b/package/crds/kubernetes.crossplane.io_objects.yaml index 06ece049..3b6faea3 100644 --- a/package/crds/kubernetes.crossplane.io_objects.yaml +++ b/package/crds/kubernetes.crossplane.io_objects.yaml @@ -10,6 +10,8 @@ spec: webhook: clientConfig: service: + name: provider-kubernetes + namespace: crossplane-system path: /convert conversionReviewVersions: - v1