Replacing slack reports to prometheus pushgateway

flipperdevices · Jan 12, 2024 · 4800e1c · 4800e1c
1 parent a274f24
commit 4800e1c
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 154 deletions.
diff --git a/cloudflare-backup.py b/cloudflare-backup.py
@@ -5,10 +5,10 @@
 import json
 import time
 import boto3
-import slack_sdk
 import CloudFlare
 from datetime import datetime
 from pathlib import Path, PurePath
+from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
 
 
 class Settings:
@@ -21,31 +21,37 @@ def __init__(self):
         self.backup_aws_bucket = os.environ.get("BACKUP_AWS_BUCKET")
         self.backup_aws_access_key = os.environ.get("BACKUP_AWS_ACCESS_KEY")
         self.backup_aws_secret_key = os.environ.get("BACKUP_AWS_SECRET_KEY")
-        self.backup_slack_token = os.environ.get("BACKUP_SLACK_TOKEN")
-        self.backup_slack_channel_success = os.environ.get(
-            "BACKUP_SLACK_CHANNEL_SUCCESS"
-        )
-        self.backup_slack_channel_fail = os.environ.get("BACKUP_SLACK_CHANNEL_FAIL")
+        self.backup_pushgateway_url = os.environ.get("BACKUP_PUSHGATEWAY_URL")
 
 
-class SlackReport:
+class PushGatewayReport:
     def __init__(self, settings: Settings):
         self.settings = settings
-        self.success_message_template = "Success backup {fqdn}!\nBackup time: {backup_time:.3f}s, upload time: {upload_time:.3f}s"
-        self.fail_message_template = "Failed to backup {fqdn}!"
-
-    def report(self, success: bool, backup_time: float = 0.0, upload_time: float = 0.0):
-        slack_client = slack_sdk.WebClient(token=self.settings.backup_slack_token)
-        fqdn = f"{self.settings.hostname}.{self.settings.namespace}.svc.cluster.local"
-        if success:
-            slack_channel = self.settings.backup_slack_channel_success
-            message = self.success_message_template.format(
-                fqdn=fqdn, backup_time=backup_time, upload_time=upload_time
-            )
-        else:
-            slack_channel = self.settings.backup_slack_channel_fail
-            message = self.fail_message_template.format(fqdn=fqdn)
-        slack_client.chat_postMessage(channel="#" + slack_channel, text=message)
+
+    def __is_reports_enabled__(self) -> bool:
+        return bool(self.settings.backup_pushgateway_url)
+
+    def report(self, backup_time: float, upload_time: float) -> None:
+        if not self.__is_reports_enabled__():
+            print("Prometheus reports are disabled!")
+            print(f"Backup time: {backup_time:.3f} s")
+            print(f"Upload time: {upload_time:.3f} s")
+            return
+        registry = CollectorRegistry()
+        backup_time_gauge = Gauge("backup_time", "backup_time", registry=registry)
+        upload_time_gauge = Gauge("upload_time", "upload_time", registry=registry)
+        backup_time_gauge.set(backup_time)
+        upload_time_gauge.set(upload_time)
+        labels = {
+            "hostname": self.settings.hostname,
+            "namespace": self.settings.namespace,
+        }
+        push_to_gateway(
+            self.settings.backup_pushgateway_url,
+            job="flipper-k8s-db-backuper",
+            registry=registry,
+            grouping_key=labels,
+        )
 
 
 class Backup:
@@ -107,16 +113,10 @@ def run(self):
 
 def main():
     settings = Settings()
-    slack_report = SlackReport(settings)
+    pushgateway_report = PushGatewayReport(settings)
     backup = Backup(settings)
-    try:
-        backup_time, upload_time = backup.run()
-        slack_report.report(
-            success=True, backup_time=backup_time, upload_time=upload_time
-        )
-    except Exception as e:
-        slack_report.report(success=False)
-        raise e
+    backup_time, upload_time = backup.run()
+    pushgateway_report.report(backup_time=backup_time, upload_time=upload_time)
 
 
 if __name__ == "__main__":

diff --git a/mongo-backup.sh b/mongo-backup.sh
@@ -3,9 +3,7 @@
 set -euo pipefail; # bash unofficial strict mode
 
 # public
-BACKUP_SLACK_TOKEN="${BACKUP_SLACK_TOKEN:-""}";
-BACKUP_SLACK_CHANNEL_SUCCESS="${BACKUP_SLACK_CHANNEL_SUCCESS:-""}";
-BACKUP_SLACK_CHANNEL_FAIL="${BACKUP_SLACK_CHANNEL_FAIL:-""}";
+BACKUP_PUSHGATEWAY_URL="${BACKUP_PUSHGATEWAY_URL:-""}";
 BACKUP_MONGO_GZIP="${BACKUP_MONGO_GZIP:-""}";
 
 # public required
@@ -23,7 +21,6 @@ BACKUP_MONGO_GZIP="${BACKUP_MONGO_GZIP:-""}";
 
 # private
 BACKUP_DUMP_BASEDIR="/backup";
-BACKUP_SLACK_MESSAGE_URL="https://slack.com/api/chat.postMessage";
 BACKUP_DUMP_DIRECTORY="$BACKUP_MONGO_NAMESPACE/$BACKUP_MONGO_HOSTNAME/$(date +%Y-%m-%d_%H-%M_%Z)";
 BACKUP_DUMP_LOCATION="$BACKUP_DUMP_BASEDIR/$BACKUP_DUMP_DIRECTORY";
 BACKUP_MONGO_FQDN="$BACKUP_MONGO_HOSTNAME.$BACKUP_MONGO_NAMESPACE.svc.cluster.local";
@@ -45,14 +42,12 @@ function create_dump() {
             --host "$BACKUP_MONGO_FQDN" \
             --port "$BACKUP_MONGO_PORT" \
             --out "$BACKUP_DUMP_LOCATION" \
-            "$GZIP_ARG_STRING" \
-            2>/dev/null;
+            "$GZIP_ARG_STRING";
     elif [[ -n "${BACKUP_MONGO_URI:-""}" ]]; then
         mongodump \
             "$BACKUP_MONGO_URI" \
             --out "$BACKUP_DUMP_LOCATION" \
-            "$GZIP_ARG_STRING" \
-            2>/dev/null;
+            "$GZIP_ARG_STRING";
     else
         echo "BACKUP_MONGO_PORT or BACKUP_MONGO_PORT required!";
         exit 1;
@@ -68,50 +63,37 @@ function upload_to_s3() {
 }
 
 function __is_reports_enabled__() {
-    if [[ -z "$BACKUP_SLACK_TOKEN" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_SUCCESS" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_FAIL" ]]; then
+    if [[ -z "$BACKUP_PUSHGATEWAY_URL" ]]; then
         return 1;
     else
         return 0;
     fi
 }
 
-function __report_to_slack__() {
-    local TEXT;
-    local DATA;
-    local SLACK_CHANNEL;
-    SLACK_CHANNEL="$1";
-    TEXT="$2";
-    if ! __is_reports_enabled__; then
-        echo "Slack reports disabled!";
-        echo -e "$TEXT";
-        return;
-    fi
-    DATA="{\"channel\":\"$SLACK_CHANNEL\",\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"$TEXT\"}}]}";
-    curl --data "$DATA" \
-        -H "Authorization: Bearer $BACKUP_SLACK_TOKEN" \
-        -H "Content-type: application/json" \
-        -X POST "$BACKUP_SLACK_MESSAGE_URL";
-}
-
-function report_fail() {
-    TEXT="Failed to backup $BACKUP_MONGO_FQDN!";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_FAIL" "$TEXT";
-    exit 1;
-}
-
-function report_success() {
+function report_to_prom() {
     local BACKUP_TIME;
     local UPLOAD_TIME;
     BACKUP_TIME="$1";
     UPLOAD_TIME="$2";
-    TEXT="Success backup $BACKUP_MONGO_FQDN!";
-    TEXT+="\nBackup time: ${BACKUP_TIME}s, upload time: ${UPLOAD_TIME}s";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_SUCCESS" "$TEXT";
-    exit 0;
+    if ! __is_reports_enabled__; then
+        echo "Prometheus reports are disabled!";
+        echo "Backup time $BACKUP_TIME";
+        echo "Upload time: $UPLOAD_TIME";
+        return;
+    fi
+    echo -e "# TYPE backup_time gauge\nbackup_time $BACKUP_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_MONGO_NAMESPACE/hostname/$BACKUP_MONGO_HOSTNAME";
+    echo -e "# TYPE upload_time gauge\nupload_time $UPLOAD_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_MONGO_NAMESPACE/hostname/$BACKUP_MONGO_HOSTNAME";
 }
 
-trap report_fail EXIT;
-BACKUP_TIME="$(TIMEFORMAT='%R';time (create_dump))";
-UPLOAD_TIME="$(TIMEFORMAT='%R';time (upload_to_s3))";
-trap - EXIT;
-report_success "$BACKUP_TIME" "$UPLOAD_TIME";
+{ TIMEFORMAT='%R'; time create_dump 2>&1 ; } 2> create_dump_time.txt
+{ TIMEFORMAT='%R'; time upload_to_s3 2>&1 ; } 2> upload_to_s3_time.txt
+
+BACKUP_TIME="$(cat create_dump_time.txt)";
+UPLOAD_TIME="$(cat upload_to_s3_time.txt)";
+report_to_prom "$BACKUP_TIME" "$UPLOAD_TIME";
diff --git a/mysql-backup.sh b/mysql-backup.sh
@@ -3,9 +3,7 @@
 set -euo pipefail; # bash unofficial strict mode
 
 # public
-BACKUP_SLACK_TOKEN=${BACKUP_SLACK_TOKEN:-""};
-BACKUP_SLACK_CHANNEL_SUCCESS=${BACKUP_SLACK_CHANNEL_SUCCESS:-""};
-BACKUP_SLACK_CHANNEL_FAIL=${BACKUP_SLACK_CHANNEL_FAIL:-""};
+BACKUP_PUSHGATEWAY_URL="${BACKUP_PUSHGATEWAY_URL:-""}";
 
 # public required
 #BACKUP_MYSQL_USER=
@@ -21,7 +19,6 @@ BACKUP_SLACK_CHANNEL_FAIL=${BACKUP_SLACK_CHANNEL_FAIL:-""};
 # private
 BACKUP_DUMP_BASEDIR="/backup";
 BACKUP_DUMP_NAME="mysqldump_all_databases.sql";
-BACKUP_SLACK_MESSAGE_URL="https://slack.com/api/chat.postMessage";
 BACKUP_DUMP_DIRECTORY="$BACKUP_MYSQL_NAMESPACE/$BACKUP_MYSQL_HOSTNAME/$(date +%Y-%m-%d_%H-%M_%Z)";
 BACKUP_DUMP_LOCATION="$BACKUP_DUMP_BASEDIR/$BACKUP_DUMP_DIRECTORY";
 BACKUP_MYSQL_FQDN="$BACKUP_MYSQL_HOSTNAME.$BACKUP_MYSQL_NAMESPACE.svc.cluster.local";
@@ -47,50 +44,37 @@ function upload_to_s3() {
 }
 
 function __is_reports_enabled__() {
-    if [[ -z "$BACKUP_SLACK_TOKEN" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_SUCCESS" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_FAIL" ]]; then
+    if [[ -z "$BACKUP_PUSHGATEWAY_URL" ]]; then
         return 1;
     else
         return 0;
     fi
 }
 
-function __report_to_slack__() {
-    local TEXT;
-    local DATA;
-    local SLACK_CHANNEL;
-    SLACK_CHANNEL="$1";
-    TEXT="$2";
-    if ! __is_reports_enabled__; then
-        echo "Slack reports disabled!";
-        echo -e "$TEXT";
-        return;
-    fi
-    DATA="{\"channel\":\"$SLACK_CHANNEL\",\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"$TEXT\"}}]}";
-    curl --data "$DATA" \
-        -H "Authorization: Bearer $BACKUP_SLACK_TOKEN" \
-        -H "Content-type: application/json" \
-        -X POST "$BACKUP_SLACK_MESSAGE_URL";
-}
-
-function report_fail() {
-    TEXT="Failed to backup $BACKUP_MYSQL_FQDN!";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_FAIL" "$TEXT";
-    exit 1;
-}
-
-function report_success() {
+function report_to_prom() {
     local BACKUP_TIME;
     local UPLOAD_TIME;
     BACKUP_TIME="$1";
     UPLOAD_TIME="$2";
-    TEXT="Success backup $BACKUP_MYSQL_FQDN!";
-    TEXT+="\nBackup time: ${BACKUP_TIME}s, upload time: ${UPLOAD_TIME}s";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_SUCCESS" "$TEXT";
-    exit 0;
+    if ! __is_reports_enabled__; then
+        echo "Prometheus reports are disabled!";
+        echo "Backup time $BACKUP_TIME";
+        echo "Upload time: $UPLOAD_TIME";
+        return;
+    fi
+    echo -e "# TYPE backup_time gauge\nbackup_time $BACKUP_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_MYSQL_NAMESPACE/hostname/$BACKUP_MYSQL_HOSTNAME";
+    echo -e "# TYPE upload_time gauge\nupload_time $UPLOAD_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_MYSQL_NAMESPACE/hostname/$BACKUP_MYSQL_HOSTNAME";
 }
 
-trap report_fail EXIT;
-BACKUP_TIME="$(TIMEFORMAT='%R';time (create_dump) 2>&1 1>/dev/null)";
-UPLOAD_TIME="$(TIMEFORMAT='%R';time (upload_to_s3) 2>&1 1>/dev/null)";
-trap - EXIT;
-report_success "$BACKUP_TIME" "$UPLOAD_TIME";
+{ TIMEFORMAT='%R'; time create_dump 2>&1 ; } 2> create_dump_time.txt
+{ TIMEFORMAT='%R'; time upload_to_s3 2>&1 ; } 2> upload_to_s3_time.txt
+
+BACKUP_TIME="$(cat create_dump_time.txt)";
+UPLOAD_TIME="$(cat upload_to_s3_time.txt)";
+report_to_prom "$BACKUP_TIME" "$UPLOAD_TIME";
diff --git a/psql-backup.sh b/psql-backup.sh
@@ -3,9 +3,7 @@
 set -euo pipefail; # bash unofficial strict mode
 
 # public
-BACKUP_SLACK_TOKEN=${BACKUP_SLACK_TOKEN:-""};
-BACKUP_SLACK_CHANNEL_SUCCESS=${BACKUP_SLACK_CHANNEL_SUCCESS:-""};
-BACKUP_SLACK_CHANNEL_FAIL=${BACKUP_SLACK_CHANNEL_FAIL:-""};
+BACKUP_PUSHGATEWAY_URL="${BACKUP_PUSHGATEWAY_URL:-""}";
 BACKUP_POSTGRESQL_PASSWORD=${BACKUP_POSTGRESQL_PASSWORD:-""};
 
 # public required
@@ -22,7 +20,6 @@ BACKUP_POSTGRESQL_PASSWORD=${BACKUP_POSTGRESQL_PASSWORD:-""};
 # private
 BACKUP_DUMP_BASEDIR="/backup";
 BACKUP_DUMP_NAME="psql_dumpall.sql";
-BACKUP_SLACK_MESSAGE_URL="https://slack.com/api/chat.postMessage";
 BACKUP_DUMP_DIRECTORY="$BACKUP_POSTGRESQL_NAMESPACE/$BACKUP_POSTGRESQL_HOSTNAME/$(date +%Y-%m-%d_%H-%M_%Z)";
 BACKUP_DUMP_LOCATION="$BACKUP_DUMP_BASEDIR/$BACKUP_DUMP_DIRECTORY";
 BACKUP_POSTGRESQL_FQDN="$BACKUP_POSTGRESQL_HOSTNAME.$BACKUP_POSTGRESQL_NAMESPACE.svc.cluster.local";
@@ -50,51 +47,37 @@ function upload_to_s3() {
 }
 
 function __is_reports_enabled__() {
-    if [[ -z "$BACKUP_SLACK_TOKEN" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_SUCCESS" ]] || [[ -z "$BACKUP_SLACK_CHANNEL_FAIL" ]]; then
+    if [[ -z "$BACKUP_PUSHGATEWAY_URL" ]]; then
         return 1;
     else
         return 0;
     fi
 }
 
-function __report_to_slack__() {
-    local TEXT;
-    local DATA;
-    local SLACK_CHANNEL;
-    SLACK_CHANNEL="$1";
-    TEXT="$2";
-    if ! __is_reports_enabled__; then
-        echo "Slack reports disabled!";
-        echo -e "$TEXT";
-        return;
-    fi
-    DATA="{\"channel\":\"$SLACK_CHANNEL\",\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"$TEXT\"}}]}";
-    curl --data "$DATA" \
-        -H "Authorization: Bearer $BACKUP_SLACK_TOKEN" \
-        -H "Content-type: application/json" \
-        -X POST "$BACKUP_SLACK_MESSAGE_URL";
-}
-
-function report_fail() {
-    TEXT="Failed to backup $BACKUP_POSTGRESQL_FQDN!";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_FAIL" "$TEXT";
-    exit 1;
-}
-
-function report_success() {
+function report_to_prom() {
     local BACKUP_TIME;
     local UPLOAD_TIME;
     BACKUP_TIME="$1";
     UPLOAD_TIME="$2";
-    TEXT="Success backup $BACKUP_POSTGRESQL_FQDN!";
-    TEXT+="\nBackup time: ${BACKUP_TIME}s, upload time: ${UPLOAD_TIME}s";
-    __report_to_slack__ "$BACKUP_SLACK_CHANNEL_SUCCESS" "$TEXT";
-    exit 0;
+    if ! __is_reports_enabled__; then
+        echo "Prometheus reports are disabled!";
+        echo "Backup time $BACKUP_TIME";
+        echo "Upload time: $UPLOAD_TIME";
+        return;
+    fi
+    echo -e "# TYPE backup_time gauge\nbackup_time $BACKUP_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_POSTGRESQL_NAMESPACE/hostname/$BACKUP_POSTGRESQL_HOSTNAME";
+    echo -e "# TYPE upload_time gauge\nupload_time $UPLOAD_TIME" | \
+        curl --fail-with-body \
+        --data-binary @- \
+        "$BACKUP_PUSHGATEWAY_URL/metrics/job/flipper-k8s-db-backuper/namespace/$BACKUP_POSTGRESQL_NAMESPACE/hostname/$BACKUP_POSTGRESQL_HOSTNAME";
 }
 
-trap report_fail EXIT;
-install_posgresql_client;
-BACKUP_TIME="$(TIMEFORMAT='%R';time (create_dump) 2>&1 1>/dev/null)";
-UPLOAD_TIME="$(TIMEFORMAT='%R';time (upload_to_s3) 2>&1 1>/dev/null)";
-trap - EXIT;
-report_success "$BACKUP_TIME" "$UPLOAD_TIME";
+{ TIMEFORMAT='%R'; time create_dump 2>&1 ; } 2> create_dump_time.txt
+{ TIMEFORMAT='%R'; time upload_to_s3 2>&1 ; } 2> upload_to_s3_time.txt
+
+BACKUP_TIME="$(cat create_dump_time.txt)";
+UPLOAD_TIME="$(cat upload_to_s3_time.txt)";
+report_to_prom "$BACKUP_TIME" "$UPLOAD_TIME";
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
 cloudflare==2.11.6
 boto3==1.28.24
-slack-sdk==3.21.3
+prometheus-client==0.19.0