Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extend Prometheus Labels to include tags (requires restart for NEW labels on the monitor to be visible) #4704

Merged
merged 4 commits into from
Aug 24, 2024
Merged

Extend Prometheus Labels to include tags (requires restart for NEW labels on the monitor to be visible) #4704

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a8c5350
dynamic prometheus labels
spali Jan 19, 2023
6c21c04
remove added debug messages
spali Apr 22, 2024
8b9a5df
Fixed typo in `server/prometheus.js`
CommanderStorm Aug 24, 2024
d9a6f00
Merge branch 'master' into dynamic_prometheus_labels
CommanderStorm Aug 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions server/model/monitor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ class Monitor extends BeanModel {
let previousBeat = null;
let retries = 0;

this.prometheus = new Prometheus(this);
this.prometheus = await Prometheus.createAndInitMetrics(this);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It could possibly throws an exception:

Trace: Error: Invalid label name
    at async Proxy.start (C:\Users\louis\Dropbox\My Projects\uptime-kuma\server\model\monitor.js:348:27)
    at async startMonitors (C:\Users\louis\Dropbox\My Projects\uptime-kuma\server\server.js:1801:9)
    at process.unexpectedErrorHandler (C:\Users\louis\Dropbox\My Projects\uptime-kuma\server\server.js:1855:13)
    at process.emit (node:events:520:28)
    at emitUnhandledRejection (node:internal/process/promises:250:13)
    at throwUnhandledRejectionsMode (node:internal/process/promises:385:19)
    at processPromiseRejections (node:internal/process/promises:470:17)
    at process.processTicksAndRejections (node:internal/process/task_queues:96:32)
If you keep encountering errors, please report to https://github.com/louislam/uptime-kuma/issues


const beat = async () => {

Expand Down Expand Up @@ -980,7 +980,7 @@ class Monitor extends BeanModel {
await R.store(bean);

log.debug("monitor", `[${this.name}] prometheus.update`);
this.prometheus?.update(bean, tlsInfo);
await this.prometheus?.update(bean, tlsInfo);

previousBeat = bean;

Expand Down
144 changes: 108 additions & 36 deletions server/prometheus.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
const { R } = require("redbean-node");
const PrometheusClient = require("prom-client");
const { log } = require("../src/util");

Expand All @@ -9,43 +10,115 @@ const commonLabels = [
"monitor_port",
];

const monitorCertDaysRemaining = new PrometheusClient.Gauge({
name: "monitor_cert_days_remaining",
help: "The number of days remaining until the certificate expires",
labelNames: commonLabels
});

const monitorCertIsValid = new PrometheusClient.Gauge({
name: "monitor_cert_is_valid",
help: "Is the certificate still valid? (1 = Yes, 0= No)",
labelNames: commonLabels
});
const monitorResponseTime = new PrometheusClient.Gauge({
name: "monitor_response_time",
help: "Monitor Response Time (ms)",
labelNames: commonLabels
});

const monitorStatus = new PrometheusClient.Gauge({
name: "monitor_status",
help: "Monitor Status (1 = UP, 0= DOWN, 2= PENDING, 3= MAINTENANCE)",
labelNames: commonLabels
});

class Prometheus {
monitorLabelValues = {};

/**
* @param {object} monitor Monitor object to monitor
* Metric: monitor_cert_days_remaining
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorCertDaysRemaining = null;

/**
* Metric: monitor_cert_is_valid
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorCertIsValid = null;

/**
* Metric: monitor_response_time
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorResponseTime = null;

/**
* Metric: monitor_status
* @type {PrometheusClient.Gauge<string> | null}
*/
static monitorStatus = null;

/**
* All registered metric labels.
* @type {string[] | null}
*/
static monitorLabelNames = null;

/**
* Monitor labels/values combination.
* @type {{}}
*/
monitorLabelValues;

/**
* Initialize metrics and get all label names the first time called.
* @returns {void}
*/
constructor(monitor) {
static async initMetrics() {
if (!this.monitorLabelNames) {
let labelNames = await R.getCol("SELECT name FROM tag");
this.monitorLabelNames = [ ...commonLabels, ...labelNames ];
}
if (!this.monitorCertDaysRemaining) {
this.monitorCertDaysRemaining = new PrometheusClient.Gauge({
name: "monitor_cert_days_remaining",
help: "The number of days remaining until the certificate expires",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorCertIsValid) {
this.monitorCertIsValid = new PrometheusClient.Gauge({
name: "monitor_cert_is_valid",
help: "Is the certificate still valid? (1 = Yes, 0 = No)",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorResponseTime) {
this.monitorResponseTime = new PrometheusClient.Gauge({
name: "monitor_response_time",
help: "Monitor Response Time (ms)",
labelNames: this.monitorLabelNames
});
}
if (!this.monitorStatus) {
this.monitorStatus = new PrometheusClient.Gauge({
name: "monitor_status",
help: "Monitor Status (1 = UP, 0 = DOWN, 2 = PENDING, 3 = MAINTENANCE)",
labelNames: this.monitorLabelNames
});
}
}

/**
* Wrapper to create a `Prometheus` instance and ensure metrics are initialized.
* @param {Monitor} monitor Monitor object to monitor
* @returns {Promise<Prometheus>} `Prometheus` instance
*/
static async createAndInitMetrics(monitor) {
await Prometheus.initMetrics();
let tags = await monitor.getTags();
return new Prometheus(monitor, tags);
}

/**
* Creates a prometheus metric instance.
*
* Note: Make sure to call `Prometheus.initMetrics()` once prior creating Prometheus instances.
* @param {Monitor} monitor Monitor object to monitor
* @param {Promise<LooseObject<any>[]>} tags Tags of the monitor
*/
constructor(monitor, tags) {
this.monitorLabelValues = {
monitor_name: monitor.name,
monitor_type: monitor.type,
monitor_url: monitor.url,
monitor_hostname: monitor.hostname,
monitor_port: monitor.port
};
Object.values(tags)
// only label names that were known at first metric creation.
.filter(tag => Prometheus.monitorLabelNames.includes(tag.name))
.forEach(tag => {
this.monitorLabelValues[tag.name] = tag.value;
});
}

/**
Expand All @@ -55,7 +128,6 @@ class Prometheus {
* @returns {void}
*/
update(heartbeat, tlsInfo) {

if (typeof tlsInfo !== "undefined") {
try {
let isValid;
Expand All @@ -64,15 +136,15 @@ class Prometheus {
} else {
isValid = 0;
}
monitorCertIsValid.set(this.monitorLabelValues, isValid);
Prometheus.monitorCertIsValid.set(this.monitorLabelValues, isValid);
} catch (e) {
log.error("prometheus", "Caught error");
log.error("prometheus", e);
}

try {
if (tlsInfo.certInfo != null) {
monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
Prometheus.monitorCertDaysRemaining.set(this.monitorLabelValues, tlsInfo.certInfo.daysRemaining);
}
} catch (e) {
log.error("prometheus", "Caught error");
Expand All @@ -82,18 +154,18 @@ class Prometheus {

if (heartbeat) {
try {
monitorStatus.set(this.monitorLabelValues, heartbeat.status);
Prometheus.monitorStatus.set(this.monitorLabelValues, heartbeat.status);
} catch (e) {
log.error("prometheus", "Caught error");
log.error("prometheus", e);
}

try {
if (typeof heartbeat.ping === "number") {
monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
Prometheus.monitorResponseTime.set(this.monitorLabelValues, heartbeat.ping);
} else {
// Is it good?
monitorResponseTime.set(this.monitorLabelValues, -1);
Prometheus.monitorResponseTime.set(this.monitorLabelValues, -1);
}
} catch (e) {
log.error("prometheus", "Caught error");
Expand All @@ -108,10 +180,10 @@ class Prometheus {
*/
remove() {
try {
monitorCertDaysRemaining.remove(this.monitorLabelValues);
monitorCertIsValid.remove(this.monitorLabelValues);
monitorResponseTime.remove(this.monitorLabelValues);
monitorStatus.remove(this.monitorLabelValues);
Prometheus.monitorCertDaysRemaining?.remove(this.monitorLabelValues);
Prometheus.monitorCertIsValid?.remove(this.monitorLabelValues);
Prometheus.monitorResponseTime?.remove(this.monitorLabelValues);
Prometheus.monitorStatus?.remove(this.monitorLabelValues);
} catch (e) {
console.error(e);
}
Expand Down
Loading