feat & fix: add options for prometheus job discovery annotations, fix… (

#86) * feat & fix: add options for prometheus job discovery annotations, fix ports of standalone sts Signed-off-by: arkbriar <[email protected]> * Release v0.1.61 Signed-off-by: arkbriar <[email protected]> --------- Signed-off-by: arkbriar <[email protected]>
risingwavelabs · May 21, 2024 · ed14113 · ed14113
1 parent 7ef59ce
commit ed14113
Show file tree

Hide file tree

Showing 11 changed files with 214 additions and 73 deletions.
diff --git a/charts/risingwave/Chart.yaml b/charts/risingwave/Chart.yaml
@@ -19,7 +19,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.60
+version: 0.1.61
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/risingwave/templates/compactor-deploy.yaml b/charts/risingwave/templates/compactor-deploy.yaml
@@ -36,8 +36,11 @@ spec:
         {{- toYaml .Values.compactorComponent.podLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.monitor.annotations.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.ports.compactor.metrics }}"
+        prometheus.io/path: "/metrics"
+        {{- end }}
         {{- include "risingwave.annotations" . | nindent 8 }}
         {{- if .Values.compactorComponent.podAnnotations }}
         {{- toYaml .Values.compactorComponent.podAnnotations | nindent 8 }}

diff --git a/charts/risingwave/templates/compute-sts.yaml b/charts/risingwave/templates/compute-sts.yaml
@@ -50,8 +50,11 @@ spec:
         {{- toYaml .Values.computeComponent.podLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.monitor.annotations.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.ports.compute.metrics }}"
+        prometheus.io/path: "/metrics"
+        {{- end }}
         {{- include "risingwave.annotations" . | nindent 8 }}
         {{- if .Values.computeComponent.podAnnotations }}
         {{- toYaml .Values.computeComponent.podAnnotations | nindent 8 }}

diff --git a/charts/risingwave/templates/frontend-deploy.yaml b/charts/risingwave/templates/frontend-deploy.yaml
@@ -37,8 +37,11 @@ spec:
         {{- toYaml .Values.frontendComponent.podLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.monitor.annotations.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.ports.frontend.metrics }}"
+        prometheus.io/path: "/metrics"
+        {{- end }}
         {{- include "risingwave.annotations" . | nindent 8 }}
         {{- if .Values.frontendComponent.podAnnotations }}
         {{- toYaml .Values.frontendComponent.podAnnotations | nindent 8 }}

diff --git a/charts/risingwave/templates/meta-sts.yaml b/charts/risingwave/templates/meta-sts.yaml
@@ -48,8 +48,11 @@ spec:
         {{- toYaml .Values.metaComponent.podLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.monitor.annotations.enabled }}
         prometheus.io/scrape: "true"
         prometheus.io/port: "{{ .Values.ports.meta.metrics }}"
+        prometheus.io/path: "/metrics"
+        {{- end }}
         {{- include "risingwave.annotations" . | nindent 8 }}
         {{- if .Values.metaComponent.podAnnotations }}
         {{- toYaml .Values.metaComponent.podAnnotations | nindent 8 }}

diff --git a/charts/risingwave/templates/service.yaml b/charts/risingwave/templates/service.yaml
@@ -55,7 +55,7 @@ spec:
   ports:
   - name: svc
     port: {{ .Values.service.port }}
-    {{- if or .Values.standalone.enabled .Values.compactMode.enabled }}
+    {{- if .Values.compactMode.enabled }}
     targetPort: f-svc
     {{- else }}
     targetPort: svc

diff --git a/charts/risingwave/templates/standalone/standalone-sts.yaml b/charts/risingwave/templates/standalone/standalone-sts.yaml
@@ -47,6 +47,11 @@ spec:
         {{- toYaml .Values.standalone.podLabels | nindent 8 }}
         {{- end }}
       annotations:
+        {{- if .Values.monitor.annotations.enabled }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.ports.frontend.metrics }}"
+        prometheus.io/path: "/metrics"
+        {{- end }}
         {{- include "risingwave.annotations" . | nindent 8 }}
         {{- if .Values.standalone.podAnnotations }}
         {{- toYaml .Values.standalone.podAnnotations | nindent 8 }}
@@ -126,7 +131,7 @@ spec:
           --meta-opts=--listen-addr 127.0.0.1:{{ .Values.ports.meta.svc }}
             --advertise-addr 127.0.0.1:{{ .Values.ports.meta.svc }}
             --dashboard-host 0.0.0.0:{{ .Values.ports.meta.dashboard }}
-            --prometheus-host 0.0.0.0:{{ .Values.ports.meta.metrics }}
+            --prometheus-host 0.0.0.0:{{ .Values.ports.frontend.metrics }}
             --backend $(RW_BACKEND)
             {{- if .Values.metaStore.etcd.enabled }}
             --etcd-endpoints $(RW_ETCD_ENDPOINTS)
@@ -136,7 +141,6 @@ spec:
             {{- end }}
             --state-store $(RW_STATE_STORE)
             --data-directory $(RW_DATA_DIRECTORY)
-            --config-path /risingwave/config/risingwave.toml
         {{- if (include "risingwave.bundle.etcd.enabled" .) }}
         {{- if and (not .Values.etcd.auth.rbac.allowNoneAuthentication) .Values.etcd.auth.rbac.create }}
             --etcd-auth
@@ -145,54 +149,32 @@ spec:
         {{- end }}
         {{- end }}
         - >-
-          --compute-opts=--config-path /risingwave/config/risingwave.toml
-            --listen-addr 127.0.0.1:{{ .Values.ports.compute.svc }}
-            --prometheus-listener-addr 0.0.0.0:{{ .Values.ports.compute.metrics }}
+          --compute-opts=--listen-addr 127.0.0.1:{{ .Values.ports.compute.svc }}
+            --prometheus-listener-addr 0.0.0.0:{{ .Values.ports.frontend.metrics }}
             --advertise-addr 127.0.0.1:{{ .Values.ports.compute.svc }}
-            --async-stack-trace verbose
             --role both
             --meta-address http://127.0.0.1:{{ .Values.ports.meta.svc }}
         - >-
-          --frontend-opts=--config-path /risingwave/config/risingwave.toml
-            --listen-addr 0.0.0.0:{{ .Values.ports.frontend.svc }}
+          --frontend-opts=--listen-addr 0.0.0.0:{{ .Values.ports.frontend.svc }}
             --advertise-addr 127.0.0.1:{{ .Values.ports.frontend.svc }}
             --prometheus-listener-addr 0.0.0.0:{{ .Values.ports.frontend.metrics }}
             --health-check-listener-addr 127.0.0.1:6786
             --meta-addr http://127.0.0.1:{{ .Values.ports.meta.svc }}
         - >-
-          --compactor-opts=--config-path /risingwave/config/risingwave.toml
-            --listen-addr 127.0.0.1:{{ .Values.ports.compactor.svc }}
+          --compactor-opts=--listen-addr 127.0.0.1:{{ .Values.ports.compactor.svc }}
             --advertise-addr 127.0.0.1:{{ .Values.ports.compactor.svc }}
-            --prometheus-listener-addr 0.0.0.0:{{ .Values.ports.compactor.metrics }}
+            --prometheus-listener-addr 0.0.0.0:{{ .Values.ports.frontend.metrics }}
             --meta-address http://127.0.0.1:{{ .Values.ports.meta.svc }}
         {{- end }}
         ports:
-        - containerPort: {{ .Values.ports.meta.svc }}
-          name: m-svc
-          protocol: TCP
         - containerPort: {{ .Values.ports.meta.dashboard }}
-          name: m-dashboard
-          protocol: TCP
-        - containerPort: {{ .Values.ports.meta.metrics }}
-          name: m-metrics
+          name: dashboard
           protocol: TCP
         - containerPort: {{ .Values.ports.frontend.svc }}
-          name: f-svc
+          name: svc
           protocol: TCP
         - containerPort: {{ .Values.ports.frontend.metrics }}
-          name: f-metrics
-          protocol: TCP
-        - containerPort: {{ .Values.ports.compute.svc }}
-          name: c-svc
-          protocol: TCP
-        - containerPort: {{ .Values.ports.compute.metrics }}
-          name: c-metrics
-          protocol: TCP
-        - containerPort: {{ .Values.ports.compactor.svc }}
-          name: cp-svc
-          protocol: TCP
-        - containerPort: {{ .Values.ports.compactor.metrics }}
-          name: cp-metrics
+          name: metrics
           protocol: TCP
         envFrom:
         {{- if .Values.standalone.extraEnvVarsConfigMap }}
@@ -400,17 +382,17 @@ spec:
           timeoutSeconds: 5
           failureThreshold: 12
           tcpSocket:
-            port: f-svc
+            port: svc
         livenessProbe:
           initialDelaySeconds: 2
           periodSeconds: 10
           tcpSocket:
-            port: f-svc
+            port: svc
         readinessProbe:
           initialDelaySeconds: 2
           periodSeconds: 10
           tcpSocket:
-            port: f-svc
+            port: svc
         {{- end }}
       {{- if .Values.standalone.additionalContainers }}
       {{- toYaml .Values.standalone.additionalContainers | nindent 6 }}

diff --git a/charts/risingwave/tests/monitor_annotations_test.yaml b/charts/risingwave/tests/monitor_annotations_test.yaml
@@ -0,0 +1,157 @@
+suite: Test Monitor Annotations
+templates:
+- meta-sts.yaml
+- frontend-deploy.yaml
+- compute-sts.yaml
+- compactor-deploy.yaml
+- standalone/standalone-sts.yaml
+chart:
+  appVersion: 1.0.0
+  version: 0.0.1
+tests:
+- it: annotations shouldn't contain prometheus.io annotations when disabled (cluster)
+  templates:
+  - meta-sts.yaml
+  - frontend-deploy.yaml
+  - compute-sts.yaml
+  - compactor-deploy.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: false
+  asserts:
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/scrape"]
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/path"]
+- it: annotations shouldn't contain prometheus.io annotations when disabled (standalone)
+  templates:
+  - standalone/standalone-sts.yaml
+  set:
+    standalone:
+      enabled: true
+
+    monitor:
+      annotations:
+        enabled: false
+  asserts:
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/scrape"]
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+  - notExists:
+      path: spec.template.metadata.annotations["prometheus.io/path"]
+- it: annotations should contain prometheus.io annotations when enabled (cluster)
+  templates:
+  - meta-sts.yaml
+  - frontend-deploy.yaml
+  - compute-sts.yaml
+  - compactor-deploy.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: true
+  asserts:
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/scrape"]
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/path"]
+- it: annotations should contain prometheus.io annotations when enabled (standalone)
+  templates:
+  - standalone/standalone-sts.yaml
+  set:
+    standalone:
+      enabled: true
+
+    monitor:
+      annotations:
+        enabled: true
+  asserts:
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/scrape"]
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+  - exists:
+      path: spec.template.metadata.annotations["prometheus.io/path"]
+- it: port value should match (meta)
+  templates:
+  - meta-sts.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: true
+
+    ports:
+      meta:
+        metrics: 1234
+  asserts:
+  - equal:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+      value: "1234"
+- it: port value should match (frontend)
+  templates:
+  - frontend-deploy.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: true
+
+    ports:
+      frontend:
+        metrics: 1234
+  asserts:
+  - equal:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+      value: "1234"
+- it: port value should match (compute)
+  templates:
+  - compute-sts.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: true
+
+    ports:
+      compute:
+        metrics: 1234
+  asserts:
+  - equal:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+      value: "1234"
+- it: port value should match (compactor)
+  templates:
+  - compactor-deploy.yaml
+  set:
+    monitor:
+      annotations:
+        enabled: true
+
+    ports:
+      compactor:
+        metrics: 1234
+  asserts:
+  - equal:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+      value: "1234"
+- it: port value should match (standalone)
+  templates:
+  - standalone/standalone-sts.yaml
+  set:
+    standalone:
+      enabled: true
+
+    monitor:
+      annotations:
+        enabled: true
+
+    ports:
+      frontend:
+        metrics: 1234
+  asserts:
+  - equal:
+      path: spec.template.metadata.annotations["prometheus.io/port"]
+      value: "1234"
diff --git a/charts/risingwave/tests/service_test.yaml b/charts/risingwave/tests/service_test.yaml
@@ -133,7 +133,7 @@ tests:
         risingwave.risingwavelabs.com/component: standalone
   - equal:
       path: spec.ports[0].targetPort
-      value: f-svc
+      value: svc
 - it: service should select different pod and port for compact mode
   set:
     compactMode: