From 2d422bd0271e15e08baafc8b6964d30d512e74e4 Mon Sep 17 00:00:00 2001
From: Deimos <deimos@tildes.net>
Date: Tue, 5 Feb 2019 22:40:26 -0700
Subject: [PATCH] Add metric for max postgresql transaction ID age

I've been reading a little about PostgreSQL transaction ID wraparound
today, and how it's knocked multiple companies out of commission for
days to get it resolved. It should have almost no chance of happening on
Tildes for years, but this will let me set up some monitoring for it
now, while I'm thinking about it.

For more info:
https://blog.sentry.io/2015/07/23/transaction-id-wraparound-in-postgres.html
---
 salt/salt/prometheus/exporters/postgres_exporter.sls       | 7 +++++++
 .../exporters/prometheus_postgres_exporter.service         | 1 +
 salt/salt/prometheus/exporters/queries.yaml                | 6 ++++++
 3 files changed, 14 insertions(+)
 create mode 100644 salt/salt/prometheus/exporters/queries.yaml

diff --git a/salt/salt/prometheus/exporters/postgres_exporter.sls b/salt/salt/prometheus/exporters/postgres_exporter.sls
index 4244164..9c22869 100644
--- a/salt/salt/prometheus/exporters/postgres_exporter.sls
+++ b/salt/salt/prometheus/exporters/postgres_exporter.sls
@@ -16,6 +16,13 @@ unpack-postgres-exporter:
     - options: --strip-components=1
     - enforce_toplevel: False
 
+/opt/prometheus_postgres_exporter/queries.yaml:
+  file.managed:
+    - source: salt://prometheus/exporters/queries.yaml
+    - user: postgres
+    - group: postgres
+    - mode: 644
+
 /etc/systemd/system/prometheus_postgres_exporter.service:
   file.managed:
     - source: salt://prometheus/exporters/prometheus_postgres_exporter.service
diff --git a/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service b/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service
index 7d7b861..21d362a 100644
--- a/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service
+++ b/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service
@@ -9,6 +9,7 @@ WorkingDirectory=/opt/prometheus_postgres_exporter
 User=postgres
 Group=postgres
 Environment="DATA_SOURCE_NAME=user=postgres host=/run/postgresql/ sslmode=disable"
+Environment="PG_EXPORTER_EXTEND_QUERY_PATH=/opt/prometheus_postgres_exporter/queries.yaml"
 ExecStart=/opt/prometheus_postgres_exporter/postgres_exporter
 
 [Install]
diff --git a/salt/salt/prometheus/exporters/queries.yaml b/salt/salt/prometheus/exporters/queries.yaml
new file mode 100644
index 0000000..309b9b1
--- /dev/null
+++ b/salt/salt/prometheus/exporters/queries.yaml
@@ -0,0 +1,6 @@
+pg_txid:
+  query: "SELECT max(age(datfrozenxid)) AS max_txid_age from pg_database"
+  metrics:
+    - max_txid_age:
+        usage: "GAUGE"
+        description: "Highest transaction ID age (wraparound at 2 billion)"