From f2c0b68f783f715ca007f0b7237bf7352f982501 Mon Sep 17 00:00:00 2001
From: Deimos <deimos@tildes.net>
Date: Fri, 3 Apr 2020 17:49:20 -0600
Subject: [PATCH] Monitoring server: add blackbox exporter

This is a prometheus exporter that allows checking IPv4 and IPv6
responses, among other things. This sets it up to make sure that the
site is responding over both IPv4 and IPv6, so that I can monitor and
set up an alert if either stops working.
---
 salt/salt/prometheus/exporters/blackbox.yml   | 14 ++++++++
 .../exporters/blackbox_exporter.sls           | 35 +++++++++++++++++++
 .../prometheus_blackbox_exporter.service      | 14 ++++++++
 salt/salt/prometheus/prometheus.yml.jinja2    | 32 +++++++++++++++++
 salt/salt/top.sls                             |  1 +
 5 files changed, 96 insertions(+)
 create mode 100644 salt/salt/prometheus/exporters/blackbox.yml
 create mode 100644 salt/salt/prometheus/exporters/blackbox_exporter.sls
 create mode 100644 salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service

diff --git a/salt/salt/prometheus/exporters/blackbox.yml b/salt/salt/prometheus/exporters/blackbox.yml
new file mode 100644
index 0000000..33cf2cb
--- /dev/null
+++ b/salt/salt/prometheus/exporters/blackbox.yml
@@ -0,0 +1,14 @@
+modules:
+  site_ipv4:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip4"
+      ip_protocol_fallback: false
+
+  site_ipv6:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip6"
+      ip_protocol_fallback: false
diff --git a/salt/salt/prometheus/exporters/blackbox_exporter.sls b/salt/salt/prometheus/exporters/blackbox_exporter.sls
new file mode 100644
index 0000000..42884b8
--- /dev/null
+++ b/salt/salt/prometheus/exporters/blackbox_exporter.sls
@@ -0,0 +1,35 @@
+# Download/extract and set up the blackbox exporter
+include:
+  - prometheus.user
+
+unpack-blackbox-exporter:
+  archive.extracted:
+    - name: /opt/prometheus_blackbox_exporter
+    - source:
+      - salt://prometheus/exporters/blackbox_exporter-0.16.0.linux-amd64.tar.gz
+      - https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz
+    - source_hash: sha256=52d3444a518ea01f220e08eaa53eb717ef54da6724760c925ab41285d0d5a7bd
+    - if_missing: /opt/prometheus_blackbox_exporter
+    - user: prometheus
+    - group: prometheus
+    - options: --strip-components=1
+    - enforce_toplevel: False
+
+/opt/prometheus_blackbox_exporter/blackbox.yml:
+  file.managed:
+    - source: salt://prometheus/exporters/blackbox.yml
+    - user: prometheus
+    - group: prometheus
+    - mode: 644
+
+/etc/systemd/system/prometheus_blackbox_exporter.service:
+  file.managed:
+    - source: salt://prometheus/exporters/prometheus_blackbox_exporter.service
+    - user: root
+    - group: root
+    - mode: 644
+
+prometheus-blackbox-exporter-service:
+  service.running:
+    - name: prometheus_blackbox_exporter
+    - enable: True
diff --git a/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service b/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service
new file mode 100644
index 0000000..430c308
--- /dev/null
+++ b/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Prometheus Blackbox Exporter
+After=syslog.target network.target
+
+[Service]
+Type=simple
+RemainAfterExit=no
+WorkingDirectory=/opt/prometheus_blackbox_exporter
+User=prometheus
+Group=prometheus
+ExecStart=/opt/prometheus_blackbox_exporter/blackbox_exporter --config.file=/opt/prometheus_blackbox_exporter/blackbox.yml
+
+[Install]
+WantedBy=multi-user.target
diff --git a/salt/salt/prometheus/prometheus.yml.jinja2 b/salt/salt/prometheus/prometheus.yml.jinja2
index 2e6ce0f..5fa35e4 100644
--- a/salt/salt/prometheus/prometheus.yml.jinja2
+++ b/salt/salt/prometheus/prometheus.yml.jinja2
@@ -23,3 +23,35 @@ scrape_configs:
     tls_config:
       insecure_skip_verify: true
     {% endif %}
+
+  # use blackbox exporter to check site is responding on IPv4
+  - job_name: "site_ipv4"
+    metrics_path: /probe
+    params:
+      module: [site_ipv4]
+    static_configs:
+      - targets:
+        - https://{{ pillar['site_hostname'] }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port
+
+  # use blackbox exporter to check site is responding on IPv6
+  - job_name: "site_ipv6"
+    metrics_path: /probe
+    params:
+      module: [site_ipv6]
+    static_configs:
+      - targets:
+        - https://{{ pillar['site_hostname'] }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port
diff --git a/salt/salt/top.sls b/salt/salt/top.sls
index eb09da0..c689fad 100644
--- a/salt/salt/top.sls
+++ b/salt/salt/top.sls
@@ -42,3 +42,4 @@ base:
     - sentry
     - grafana
     - prometheus
+    - prometheus.exporters.blackbox_exporter