Salt: update states related to Prometheus

Some of these states were built entirely around a single-server approach
(Prometheus + monitoring being on the same server as the site), and the
files have needed modifications to work with a separate monitoring
server.

This updates the states so that it should all happen as expected in all
types of environments.

salt/pillar/dev.sls

@@ -3,3 +3,5 @@ ssl_cert_path: /etc/pki/tls/certs/localhost.crt
 ssl_private_key_path: /etc/pki/tls/certs/localhost.key
 nginx_worker_processes: 1
 postgresql_version: 10
+prometheus_ips: ['127.0.0.1']
+site_hostname: localhost

salt/pillar/monitoring.sls

@@ -1,5 +1,6 @@
-ssl_cert_path: /etc/pki/tls/certs/localhost.crt
-ssl_private_key_path: /etc/pki/tls/certs/localhost.key
+ssl_cert_path: /etc/letsencrypt/live/tildes.net/fullchain.pem
+ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem
 hsts_max_age: 60
 nginx_worker_processes: auto
 postgresql_version: 9.6
+site_hostname: tildes.net

salt/pillar/prod.sls

@@ -4,3 +4,5 @@ ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem
 hsts_max_age: 63072000
 nginx_worker_processes: auto
 postgresql_version: 10
+prometheus_ips: ['127.0.0.1']
+site_hostname: tildes.net

salt/salt/nginx/tildes.conf.jinja2

@@ -56,15 +56,17 @@ server {
     add_header X-Xss-Protection "1; mode=block" always;
     add_header Referrer-Policy "same-origin" always;
 
-    server_name tildes.net localhost;
+    server_name {{ pillar['site_hostname'] }};
 
     keepalive_timeout 5;
 
     root {{ app_dir }}/static;
 
-    # Block access to /metrics except from localhost (for Prometheus)
+    # Block access to /metrics except from Prometheus server(s)
     location /metrics {
-        allow 127.0.0.1;
+        {% for ip in pillar['prometheus_ips'] %}
+        allow {{ ip }};
+        {% endfor %}
         deny all;
 
         # try_files is unnecessary here, but I don't know the "proper" way

salt/salt/prometheus/init.sls

@@ -31,7 +31,8 @@ prometheus-service:
 
 /opt/prometheus/prometheus.yml:
   file.managed:
-    - source: salt://prometheus/prometheus.yml
+    - source: salt://prometheus/prometheus.yml.jinja2
+    - template: jinja
     - user: prometheus
     - group: prometheus
     - mode: 664

salt/salt/prometheus/prometheus.yml → salt/salt/prometheus/prometheus.yml.jinja2

@@ -5,23 +5,25 @@ global:
 scrape_configs:
   - job_name: "node"
     static_configs:
-      - targets: ['localhost:9100']
+      - targets: ['{{ pillar['site_hostname'] }}:9100']
 
   - job_name: "rabbitmq"
     static_configs:
-      - targets: ['localhost:9419']
+      - targets: ['{{ pillar['site_hostname'] }}:9419']
 
   - job_name: "redis"
     static_configs:
-      - targets: ['localhost:9121']
+      - targets: ['{{ pillar['site_hostname'] }}:9121']
 
   - job_name: "postgres"
     static_configs:
-      - targets: ['localhost:9187']
+      - targets: ['{{ pillar['site_hostname'] }}:9187']
 
   - job_name: "tildes"
     scheme: https
     static_configs:
-      - targets: ['localhost:443']
+      - targets: ['{{ pillar['site_hostname'] }}:443']
+    {% if grains['id'] == 'dev' %}
     tls_config:
       insecure_skip_verify: true
+    {% endif %}