From 4cc100ab02eb14c4f5d058a829970e735d91690c Mon Sep 17 00:00:00 2001 From: Deimos Date: Fri, 25 Jun 2021 00:08:58 -0600 Subject: [PATCH] Switch to Debian 10 and Ansible This changes the site to run on Debian 10 instead of Ubuntu 16.04. It also fully converts the previous Salt setup to use Ansible instead. Most of this was a relatively straightforward conversion, and it should be very close to equivalent. One notable difference is that I removed the setup for the "monitoring" server, since I wasn't confident that the way of setting up self-hosted Sentry and Grafana was working any more. I'll look to re-add that at some point, but it's not urgent. --- Vagrantfile | 36 +++-- ansible/common.yml | 9 ++ ansible/group_vars/dev.yml | 33 ++++ ansible/group_vars/prod.yml | 52 ++++++ ansible/playbook.yml | 54 +++++++ ansible/requirements.yml | 3 + ansible/roles/boussole/defaults/main.yml | 2 + ansible/roles/boussole/tasks/main.yml | 44 ++++++ .../templates}/boussole.service.jinja2 | 3 +- ansible/roles/cmark-gfm/tasks/main.yml | 40 +++++ ansible/roles/common/tasks/main.yml | 13 ++ ansible/roles/consumers/defaults/main.yml | 6 + ansible/roles/consumers/meta/main.yml | 4 + ansible/roles/consumers/tasks/main.yml | 16 ++ .../templates/consumer.service.jinja2 | 7 +- ansible/roles/cronjobs/tasks/main.yml | 53 +++++++ .../roles/development/files/ipython_config.py | 2 + ansible/roles/development/tasks/main.yml | 26 +++ .../gunicorn/files}/gunicorn_reloader.path | 0 .../gunicorn/files}/gunicorn_reloader.service | 0 ansible/roles/gunicorn/meta/main.yml | 3 + ansible/roles/gunicorn/tasks/main.yml | 54 +++++++ .../gunicorn/templates}/gunicorn.conf.jinja2 | 1 - .../templates}/gunicorn.service.jinja2 | 3 +- .../templates}/gunicorn.socket.jinja2 | 1 - .../roles/ipv6_networking/defaults/main.yml | 2 + ansible/roles/ipv6_networking/tasks/main.yml | 23 +++ ansible/roles/java/tasks/main.yml | 4 + ansible/roles/nginx/defaults/main.yml | 6 + .../roles/nginx/files}/logrotate | 1 + ansible/roles/nginx/handlers/main.yml | 5 + ansible/roles/nginx/tasks/main.yml | 52 ++++++ .../roles/nginx/templates}/nginx.conf.jinja2 | 13 +- .../roles/nginx_prod_config/defaults/main.yml | 3 + ansible/roles/nginx_prod_config/meta/main.yml | 4 + .../roles/nginx_prod_config/tasks/main.yml | 38 +++++ .../templates}/tildes-shortener.conf.jinja2 | 3 +- .../tildes-static-sites.conf.jinja2 | 4 +- .../roles/nginx_site_config/defaults/main.yml | 11 ++ ansible/roles/nginx_site_config/meta/main.yml | 4 + .../roles/nginx_site_config/tasks/main.yml | 21 +++ .../templates}/tildes.conf.jinja2 | 21 +-- ansible/roles/nodejs/tasks/main.yml | 20 +++ ansible/roles/pgbouncer/handlers/main.yml | 5 + ansible/roles/pgbouncer/meta/main.yml | 3 + ansible/roles/pgbouncer/tasks/main.yml | 31 ++++ .../pgbouncer/templates}/pgbouncer.ini.jinja2 | 5 +- ansible/roles/postgresql/defaults/main.yml | 12 ++ ansible/roles/postgresql/handlers/main.yml | 10 ++ ansible/roles/postgresql/tasks/main.yml | 27 ++++ .../roles/postgresql_plpython/meta/main.yml | 3 + .../roles/postgresql_plpython/tasks/main.yml | 12 ++ .../postgresql_redis_bridge/meta/main.yml | 4 + .../postgresql_redis_bridge/tasks/main.yml | 13 ++ .../postgresql_redis_bridge.service.jinja2 | 3 +- .../postgresql_tildes_dbs/defaults/main.yml | 6 + .../roles/postgresql_tildes_dbs/meta/main.yml | 8 + .../postgresql_tildes_dbs/tasks/database.yml | 38 +++++ .../postgresql_tildes_dbs/tasks/main.yml | 62 ++++++++ ansible/roles/prometheus/defaults/main.yml | 8 + .../prometheus/files}/prometheus.service | 0 ansible/roles/prometheus/handlers/main.yml | 5 + ansible/roles/prometheus/tasks/main.yml | 59 +++++++ .../templates}/prometheus.yml.jinja2 | 42 ++--- .../files}/prometheus_node_exporter.service | 0 .../prometheus_node_exporter/tasks/main.yml | 42 +++++ .../prometheus_postgres_exporter.service | 0 .../files}/queries.yaml | 0 .../meta/main.yml | 3 + .../tasks/main.yml | 47 ++++++ .../files}/prometheus_redis_exporter.service | 0 .../prometheus_redis_exporter/meta/main.yml | 3 + .../prometheus_redis_exporter/tasks/main.yml | 40 +++++ ansible/roles/pts_lbsearch/tasks/main.yml | 13 ++ ansible/roles/python/defaults/main.yml | 2 + ansible/roles/python/tasks/main.yml | 93 +++++++++++ ansible/roles/redis/defaults/main.yml | 2 + .../roles/redis/files}/redis.service | 0 .../redis/files}/transparent_hugepage.service | 0 ansible/roles/redis/handlers/main.yml | 5 + ansible/roles/redis/tasks/main.yml | 131 ++++++++++++++++ .../roles/redis/templates}/redis.conf.jinja2 | 6 - ansible/roles/redis_module_cell/meta/main.yml | 3 + .../roles/redis_module_cell/tasks/main.yml | 27 ++++ ansible/roles/scripts/tasks/main.yml | 8 + .../scripts/templates}/activate.sh.jinja2 | 1 - .../roles/self_signed_ssl_cert/meta/main.yml | 3 + .../roles/self_signed_ssl_cert/tasks/main.yml | 23 +++ ansible/roles/webassets/meta/main.yml | 3 + ansible/roles/webassets/tasks/main.yml | 29 ++++ .../templates}/webassets.service.jinja2 | 1 - ansible/roles/wiki_repo/tasks/main.yml | 24 +++ ansible/tasks/prometheus_user.yml | 10 ++ ansible/vars.yml | 8 + salt/minion | 19 --- salt/pillar/dev.sls | 8 - salt/pillar/monitoring-secrets.example.sls | 8 - salt/pillar/monitoring.sls | 6 - salt/pillar/prod.sls | 11 -- salt/pillar/top.sls | 8 - salt/salt/boussole.sls | 47 ------ salt/salt/cmark-gfm.sls | 25 --- salt/salt/common.jinja2 | 13 -- ...ent_user_mentions_generator.service.jinja2 | 18 --- salt/salt/consumers/init.sls | 85 ---------- ...st_processing_script_runner.service.jinja2 | 18 --- .../site_icon_downloader.service.jinja2 | 17 -- .../topic_embedly_extractor.service.jinja2 | 18 --- ...nteresting_activity_updater.service.jinja2 | 18 --- .../topic_metadata_generator.service.jinja2 | 18 --- salt/salt/cronjobs.sls | 46 ------ salt/salt/development.sls | 27 ---- salt/salt/final-setup.sls | 19 --- salt/salt/general-config.sls | 3 - salt/salt/grafana/grafana.conf.jinja2 | 19 --- salt/salt/grafana/grafana.ini.jinja2 | 17 -- salt/salt/grafana/init.sls | 61 -------- salt/salt/gunicorn/init.sls | 59 ------- salt/salt/java.sls | 3 - salt/salt/nginx/init.sls | 33 ---- salt/salt/nginx/shortener-config.sls | 16 -- salt/salt/nginx/site-config.sls | 16 -- salt/salt/nginx/static-sites-config.sls | 16 -- salt/salt/nodejs.sls | 25 --- salt/salt/postgresql-redis-bridge.sls | 14 -- salt/salt/postgresql/init.sls | 85 ---------- salt/salt/postgresql/pg_hba.conf.jinja2 | 6 - salt/salt/postgresql/pgbouncer.sls | 25 --- salt/salt/postgresql/prod-config.sls | 25 --- salt/salt/postgresql/site-db.sls | 73 --------- salt/salt/postgresql/test-db.sls | 58 ------- salt/salt/prod-config.sls | 17 -- salt/salt/prometheus/exporters/blackbox.yml | 14 -- .../exporters/blackbox_exporter.sls | 35 ----- .../prometheus/exporters/node_exporter.sls | 28 ---- .../exporters/postgres_exporter.sls | 36 ----- .../prometheus_blackbox_exporter.service | 14 -- .../prometheus/exporters/redis_exporter.sls | 27 ---- salt/salt/prometheus/init.sls | 58 ------- salt/salt/prometheus/prometheus.conf.jinja2 | 21 --- salt/salt/prometheus/user.sls | 7 - salt/salt/pts-lbsearch.sls | 12 -- salt/salt/python.sls | 60 ------- salt/salt/redis/init.sls | 118 -------------- salt/salt/redis/modules/redis-cell.sls | 20 --- salt/salt/scripts/init.sls | 7 - salt/salt/self-signed-cert.sls | 17 -- salt/salt/sentry/common.jinja2 | 4 - salt/salt/sentry/config.yml.jinja2 | 64 -------- salt/salt/sentry/init.sls | 148 ------------------ salt/salt/sentry/sentry-cron.service.jinja2 | 15 -- salt/salt/sentry/sentry-web.service.jinja2 | 17 -- salt/salt/sentry/sentry-worker.service.jinja2 | 15 -- salt/salt/sentry/sentry.conf.jinja2 | 21 --- salt/salt/sentry/sentry.conf.py | 135 ---------------- salt/salt/tildes-wiki.sls | 23 --- salt/salt/top.sls | 46 ------ salt/salt/webassets.sls | 25 --- 158 files changed, 1495 insertions(+), 2075 deletions(-) create mode 100644 ansible/common.yml create mode 100644 ansible/group_vars/dev.yml create mode 100644 ansible/group_vars/prod.yml create mode 100644 ansible/playbook.yml create mode 100644 ansible/requirements.yml create mode 100644 ansible/roles/boussole/defaults/main.yml create mode 100644 ansible/roles/boussole/tasks/main.yml rename {salt/salt => ansible/roles/boussole/templates}/boussole.service.jinja2 (63%) create mode 100644 ansible/roles/cmark-gfm/tasks/main.yml create mode 100644 ansible/roles/common/tasks/main.yml create mode 100644 ansible/roles/consumers/defaults/main.yml create mode 100644 ansible/roles/consumers/meta/main.yml create mode 100644 ansible/roles/consumers/tasks/main.yml rename salt/salt/consumers/topic_youtube_scraper.service.jinja2 => ansible/roles/consumers/templates/consumer.service.jinja2 (50%) create mode 100644 ansible/roles/cronjobs/tasks/main.yml create mode 100644 ansible/roles/development/files/ipython_config.py create mode 100644 ansible/roles/development/tasks/main.yml rename {salt/salt/gunicorn => ansible/roles/gunicorn/files}/gunicorn_reloader.path (100%) rename {salt/salt/gunicorn => ansible/roles/gunicorn/files}/gunicorn_reloader.service (100%) create mode 100644 ansible/roles/gunicorn/meta/main.yml create mode 100644 ansible/roles/gunicorn/tasks/main.yml rename {salt/salt/gunicorn => ansible/roles/gunicorn/templates}/gunicorn.conf.jinja2 (56%) rename {salt/salt/gunicorn => ansible/roles/gunicorn/templates}/gunicorn.service.jinja2 (61%) rename {salt/salt/gunicorn => ansible/roles/gunicorn/templates}/gunicorn.socket.jinja2 (80%) create mode 100644 ansible/roles/ipv6_networking/defaults/main.yml create mode 100644 ansible/roles/ipv6_networking/tasks/main.yml create mode 100644 ansible/roles/java/tasks/main.yml create mode 100644 ansible/roles/nginx/defaults/main.yml rename {salt/salt/nginx => ansible/roles/nginx/files}/logrotate (86%) create mode 100644 ansible/roles/nginx/handlers/main.yml create mode 100644 ansible/roles/nginx/tasks/main.yml rename {salt/salt/nginx => ansible/roles/nginx/templates}/nginx.conf.jinja2 (89%) create mode 100644 ansible/roles/nginx_prod_config/defaults/main.yml create mode 100644 ansible/roles/nginx_prod_config/meta/main.yml create mode 100644 ansible/roles/nginx_prod_config/tasks/main.yml rename {salt/salt/nginx => ansible/roles/nginx_prod_config/templates}/tildes-shortener.conf.jinja2 (84%) rename {salt/salt/nginx => ansible/roles/nginx_prod_config/templates}/tildes-static-sites.conf.jinja2 (91%) create mode 100644 ansible/roles/nginx_site_config/defaults/main.yml create mode 100644 ansible/roles/nginx_site_config/meta/main.yml create mode 100644 ansible/roles/nginx_site_config/tasks/main.yml rename {salt/salt/nginx => ansible/roles/nginx_site_config/templates}/tildes.conf.jinja2 (87%) create mode 100644 ansible/roles/nodejs/tasks/main.yml create mode 100644 ansible/roles/pgbouncer/handlers/main.yml create mode 100644 ansible/roles/pgbouncer/meta/main.yml create mode 100644 ansible/roles/pgbouncer/tasks/main.yml rename {salt/salt/postgresql => ansible/roles/pgbouncer/templates}/pgbouncer.ini.jinja2 (64%) create mode 100644 ansible/roles/postgresql/defaults/main.yml create mode 100644 ansible/roles/postgresql/handlers/main.yml create mode 100644 ansible/roles/postgresql/tasks/main.yml create mode 100644 ansible/roles/postgresql_plpython/meta/main.yml create mode 100644 ansible/roles/postgresql_plpython/tasks/main.yml create mode 100644 ansible/roles/postgresql_redis_bridge/meta/main.yml create mode 100644 ansible/roles/postgresql_redis_bridge/tasks/main.yml rename {salt/salt => ansible/roles/postgresql_redis_bridge/templates}/postgresql_redis_bridge.service.jinja2 (73%) create mode 100644 ansible/roles/postgresql_tildes_dbs/defaults/main.yml create mode 100644 ansible/roles/postgresql_tildes_dbs/meta/main.yml create mode 100644 ansible/roles/postgresql_tildes_dbs/tasks/database.yml create mode 100644 ansible/roles/postgresql_tildes_dbs/tasks/main.yml create mode 100644 ansible/roles/prometheus/defaults/main.yml rename {salt/salt/prometheus => ansible/roles/prometheus/files}/prometheus.service (100%) create mode 100644 ansible/roles/prometheus/handlers/main.yml create mode 100644 ansible/roles/prometheus/tasks/main.yml rename {salt/salt/prometheus => ansible/roles/prometheus/templates}/prometheus.yml.jinja2 (53%) rename {salt/salt/prometheus/exporters => ansible/roles/prometheus_node_exporter/files}/prometheus_node_exporter.service (100%) create mode 100644 ansible/roles/prometheus_node_exporter/tasks/main.yml rename {salt/salt/prometheus/exporters => ansible/roles/prometheus_postgres_exporter/files}/prometheus_postgres_exporter.service (100%) rename {salt/salt/prometheus/exporters => ansible/roles/prometheus_postgres_exporter/files}/queries.yaml (100%) create mode 100644 ansible/roles/prometheus_postgres_exporter/meta/main.yml create mode 100644 ansible/roles/prometheus_postgres_exporter/tasks/main.yml rename {salt/salt/prometheus/exporters => ansible/roles/prometheus_redis_exporter/files}/prometheus_redis_exporter.service (100%) create mode 100644 ansible/roles/prometheus_redis_exporter/meta/main.yml create mode 100644 ansible/roles/prometheus_redis_exporter/tasks/main.yml create mode 100644 ansible/roles/pts_lbsearch/tasks/main.yml create mode 100644 ansible/roles/python/defaults/main.yml create mode 100644 ansible/roles/python/tasks/main.yml create mode 100644 ansible/roles/redis/defaults/main.yml rename {salt/salt/redis => ansible/roles/redis/files}/redis.service (100%) rename {salt/salt/redis => ansible/roles/redis/files}/transparent_hugepage.service (100%) create mode 100644 ansible/roles/redis/handlers/main.yml create mode 100644 ansible/roles/redis/tasks/main.yml rename {salt/salt/redis => ansible/roles/redis/templates}/redis.conf.jinja2 (99%) create mode 100644 ansible/roles/redis_module_cell/meta/main.yml create mode 100644 ansible/roles/redis_module_cell/tasks/main.yml create mode 100644 ansible/roles/scripts/tasks/main.yml rename {salt/salt/scripts => ansible/roles/scripts/templates}/activate.sh.jinja2 (78%) create mode 100644 ansible/roles/self_signed_ssl_cert/meta/main.yml create mode 100644 ansible/roles/self_signed_ssl_cert/tasks/main.yml create mode 100644 ansible/roles/webassets/meta/main.yml create mode 100644 ansible/roles/webassets/tasks/main.yml rename {salt/salt => ansible/roles/webassets/templates}/webassets.service.jinja2 (80%) create mode 100644 ansible/roles/wiki_repo/tasks/main.yml create mode 100644 ansible/tasks/prometheus_user.yml create mode 100644 ansible/vars.yml delete mode 100644 salt/minion delete mode 100644 salt/pillar/dev.sls delete mode 100644 salt/pillar/monitoring-secrets.example.sls delete mode 100644 salt/pillar/monitoring.sls delete mode 100644 salt/pillar/prod.sls delete mode 100644 salt/pillar/top.sls delete mode 100644 salt/salt/boussole.sls delete mode 100644 salt/salt/cmark-gfm.sls delete mode 100644 salt/salt/common.jinja2 delete mode 100644 salt/salt/consumers/comment_user_mentions_generator.service.jinja2 delete mode 100644 salt/salt/consumers/init.sls delete mode 100644 salt/salt/consumers/post_processing_script_runner.service.jinja2 delete mode 100644 salt/salt/consumers/site_icon_downloader.service.jinja2 delete mode 100644 salt/salt/consumers/topic_embedly_extractor.service.jinja2 delete mode 100644 salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 delete mode 100644 salt/salt/consumers/topic_metadata_generator.service.jinja2 delete mode 100644 salt/salt/cronjobs.sls delete mode 100644 salt/salt/development.sls delete mode 100644 salt/salt/final-setup.sls delete mode 100644 salt/salt/general-config.sls delete mode 100644 salt/salt/grafana/grafana.conf.jinja2 delete mode 100644 salt/salt/grafana/grafana.ini.jinja2 delete mode 100644 salt/salt/grafana/init.sls delete mode 100644 salt/salt/gunicorn/init.sls delete mode 100644 salt/salt/java.sls delete mode 100644 salt/salt/nginx/init.sls delete mode 100644 salt/salt/nginx/shortener-config.sls delete mode 100644 salt/salt/nginx/site-config.sls delete mode 100644 salt/salt/nginx/static-sites-config.sls delete mode 100644 salt/salt/nodejs.sls delete mode 100644 salt/salt/postgresql-redis-bridge.sls delete mode 100644 salt/salt/postgresql/init.sls delete mode 100644 salt/salt/postgresql/pg_hba.conf.jinja2 delete mode 100644 salt/salt/postgresql/pgbouncer.sls delete mode 100644 salt/salt/postgresql/prod-config.sls delete mode 100644 salt/salt/postgresql/site-db.sls delete mode 100644 salt/salt/postgresql/test-db.sls delete mode 100644 salt/salt/prod-config.sls delete mode 100644 salt/salt/prometheus/exporters/blackbox.yml delete mode 100644 salt/salt/prometheus/exporters/blackbox_exporter.sls delete mode 100644 salt/salt/prometheus/exporters/node_exporter.sls delete mode 100644 salt/salt/prometheus/exporters/postgres_exporter.sls delete mode 100644 salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service delete mode 100644 salt/salt/prometheus/exporters/redis_exporter.sls delete mode 100644 salt/salt/prometheus/init.sls delete mode 100644 salt/salt/prometheus/prometheus.conf.jinja2 delete mode 100644 salt/salt/prometheus/user.sls delete mode 100644 salt/salt/pts-lbsearch.sls delete mode 100644 salt/salt/python.sls delete mode 100644 salt/salt/redis/init.sls delete mode 100644 salt/salt/redis/modules/redis-cell.sls delete mode 100644 salt/salt/scripts/init.sls delete mode 100644 salt/salt/self-signed-cert.sls delete mode 100644 salt/salt/sentry/common.jinja2 delete mode 100644 salt/salt/sentry/config.yml.jinja2 delete mode 100644 salt/salt/sentry/init.sls delete mode 100644 salt/salt/sentry/sentry-cron.service.jinja2 delete mode 100644 salt/salt/sentry/sentry-web.service.jinja2 delete mode 100644 salt/salt/sentry/sentry-worker.service.jinja2 delete mode 100644 salt/salt/sentry/sentry.conf.jinja2 delete mode 100644 salt/salt/sentry/sentry.conf.py delete mode 100644 salt/salt/tildes-wiki.sls delete mode 100644 salt/salt/top.sls delete mode 100644 salt/salt/webassets.sls diff --git a/Vagrantfile b/Vagrantfile index d7e9e75..dcff88d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,25 +4,39 @@ VAGRANT_CONFIG_VERSION = "2" Vagrant.configure(VAGRANT_CONFIG_VERSION) do |config| - config.vm.box = "ubuntu/xenial64" + # Using the "contrib" version for vboxsf module for synced folders + config.vm.box = "debian/contrib-buster64" # Main application folder config.vm.synced_folder "tildes/", "/opt/tildes/" - # Mount the salt file root and pillar root - config.vm.synced_folder "salt/salt/", "/srv/salt/" - config.vm.synced_folder "salt/pillar/", "/srv/pillar/" + config.vm.synced_folder "ansible/", "/srv/ansible" config.vm.network "forwarded_port", guest: 443, host: 4443 config.vm.network "forwarded_port", guest: 9090, host: 9090 - # Masterless salt provisioning - config.vm.provision :salt do |salt| - salt.masterless = true - salt.minion_config = "salt/minion" - salt.run_highstate = true - salt.verbose = true - salt.log_level = "info" + config.vm.provision "ansible_local" do |ansible| + ansible.install_mode = "pip" + + # Since Debian Buster still uses Python 2.7 by default and the pip bootstrap + # script is no longer compatible with 2.7, we need to specify the installation + # command manually. If we upgrade to a newer version of Debian that defaults to + # Python 3.6+, this should no longer be necessary. + ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl -s https://bootstrap.pypa.io/get-pip.py | sudo python3" + + # Vagrant doesn't currently recognize the new format for Ansible versions + # (e.g. "ansible [core 2.11.1]"), so the compatibility mode is set incorrectly. + # A new version of Vagrant should resolve this soon. + ansible.compatibility_mode = "2.0" + + # put the VM into the "dev" and "app_server" Ansible groups + ansible.groups = { + "dev" => ["default"], + "app_server" => ["default"], + } + + ansible.galaxy_role_file = "ansible/requirements.yml" + ansible.playbook = "ansible/playbook.yml" end config.vm.provider "virtualbox" do |vb| diff --git a/ansible/common.yml b/ansible/common.yml new file mode 100644 index 0000000..9797079 --- /dev/null +++ b/ansible/common.yml @@ -0,0 +1,9 @@ +--- +- hosts: all + become: true + + vars_files: + - vars.yml + + roles: + - common diff --git a/ansible/group_vars/dev.yml b/ansible/group_vars/dev.yml new file mode 100644 index 0000000..a265d20 --- /dev/null +++ b/ansible/group_vars/dev.yml @@ -0,0 +1,33 @@ +--- +app_username: vagrant +pip_requirements_filename: requirements-dev.txt +ini_file: development.ini + +gunicorn_args: --reload + +# have to disable sendfile for vagrant due to a virtualbox bug +nginx_enable_sendfile: false +nginx_worker_processes: 1 + +nginx_enable_hsts: false +nginx_enable_csp: false + +postgresql_tildes_databases: + - tildes + - tildes_test +postgresql_tildes_user_flags: "SUPERUSER" +tildes_database_insert_dev_data: true + +hsts_max_age: 60 + +site_hostname: localhost + +ssl_cert_dir: /etc/pki/tls/certs +ssl_cert_path: "{{ ssl_cert_dir }}/localhost.crt" +ssl_private_key_path: "{{ ssl_cert_dir }}/localhost.key" + +ansible_python_interpreter: /usr/bin/python3 + +# Workaround for some Ansible permissions issues when becoming an unprivileged user +# (this has some risks, but should be fine for our use) +ansible_shell_allow_world_readable_temp: true diff --git a/ansible/group_vars/prod.yml b/ansible/group_vars/prod.yml new file mode 100644 index 0000000..b405874 --- /dev/null +++ b/ansible/group_vars/prod.yml @@ -0,0 +1,52 @@ +--- +app_username: tildes +postgresql_settings: + checkpoint_completion_target: 0.7 + default_statistics_target: 100 + effective_cache_size: 24GB + effective_io_concurrency: 200 + maintenance_work_mem: 2GB + max_parallel_workers: 8 + max_parallel_workers_per_gather: 4 + max_wal_size: 2GB + max_worker_processes: 8 + min_wal_size: 1GB + random_page_cost: 1.1 + shared_buffers: 8GB + wal_buffers: 16MB + work_mem: 10485kB +ini_file: production.ini +gunicorn_args: --workers 8 +site_hostname: tildes.net + +# add extra prod-only consumer services (e.g. ones that use external APIs) +consumers: + - comment_user_mentions_generator + - post_processing_script_runner + - site_icon_downloader + - topic_embedly_extractor + - topic_interesting_activity_updater + - topic_metadata_generator + - topic_youtube_scraper + +ipv6_device: eno1 +ipv6_address: "2607:5300:0203:2e7a::" +ipv6_gateway: "2607:5300:0203:2eff:ff:ff:ff:ff" + +prometheus_ips: + - "2607:5300:201:3100::6e77" + +prometheus_consumer_scrape_targets: + comment_user_mentions_generator: 25010 + post_processing_script_runner: 25016 + site_icon_downloader: 25011 + topic_embedly_extractor: 25012 + topic_interesting_activity_updater: 25013 + topic_metadata_generator: 25014 + topic_youtube_scraper: 25015 + +ansible_python_interpreter: /usr/bin/python3 + +# Workaround for some Ansible permissions issues when becoming an unprivileged user +# (this has some risks, but should be fine for our use) +ansible_shell_allow_world_readable_temp: true diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 0000000..86f5dad --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,54 @@ +--- +- hosts: all + become: true + vars_files: + - vars.yml + roles: + - common + +- hosts: app_server + become: true + vars_files: + - vars.yml + roles: + - cmark-gfm + - pts_lbsearch + - python + - gunicorn + - nginx + - nginx_site_config + - postgresql + - postgresql_plpython + - postgresql_tildes_dbs + - pgbouncer + - redis + - redis_module_cell + - postgresql_redis_bridge + - boussole + - webassets + - scripts + - prometheus_node_exporter + - prometheus_postgres_exporter + - prometheus_redis_exporter + - consumers + - cronjobs + - wiki_repo + +- hosts: dev + become: true + vars_files: + - vars.yml + roles: + - self_signed_ssl_cert + - prometheus + - java + - nodejs + - development + +- hosts: prod + become: true + vars_files: + - vars.yml + roles: + - nginx_prod_config + - ipv6_networking diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..8dd5161 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - community.general diff --git a/ansible/roles/boussole/defaults/main.yml b/ansible/roles/boussole/defaults/main.yml new file mode 100644 index 0000000..f7173fb --- /dev/null +++ b/ansible/roles/boussole/defaults/main.yml @@ -0,0 +1,2 @@ +--- +boussole_venv_dir: /opt/venvs/boussole diff --git a/ansible/roles/boussole/tasks/main.yml b/ansible/roles/boussole/tasks/main.yml new file mode 100644 index 0000000..103264c --- /dev/null +++ b/ansible/roles/boussole/tasks/main.yml @@ -0,0 +1,44 @@ +--- +- name: Create venv and install boussole with pip + pip: + virtualenv: "{{ boussole_venv_dir }}" + virtualenv_command: python{{ python_version }} -m venv + name: boussole + +- name: Create systemd service file + template: + src: boussole.service.jinja2 + dest: /etc/systemd/system/boussole.service + owner: root + group: root + mode: 0644 + +- name: Start and enable boussole service + service: + name: boussole + state: started + enabled: true + +- name: Create directory for compiled CSS + file: + path: "{{ app_dir }}/static/css" + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0755 + +- name: Check if any compiled CSS files exist + find: + path: "{{ app_dir }}/static/css" + patterns: + - "*.css" + register: compiled_css_files + +- name: Run boussole manually if no compiled CSS exists yet + command: + chdir: "{{ app_dir }}" + cmd: "{{ boussole_venv_dir }}/bin/boussole compile --backend=yaml --config=boussole.yaml" + environment: + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + when: compiled_css_files.matched == 0 diff --git a/salt/salt/boussole.service.jinja2 b/ansible/roles/boussole/templates/boussole.service.jinja2 similarity index 63% rename from salt/salt/boussole.service.jinja2 rename to ansible/roles/boussole/templates/boussole.service.jinja2 index 05c418e..064066d 100644 --- a/salt/salt/boussole.service.jinja2 +++ b/ansible/roles/boussole/templates/boussole.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username -%} [Unit] Description=Boussole - auto-compile SCSS files on change @@ -7,7 +6,7 @@ User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }} Environment="LC_ALL=C.UTF-8" "LANG=C.UTF-8" -ExecStart=/opt/venvs/boussole/bin/boussole watch --backend=yaml --config=boussole.yaml --poll +ExecStart={{ boussole_venv_dir }}/bin/boussole watch --backend=yaml --config=boussole.yaml --poll Restart=always RestartSec=5 diff --git a/ansible/roles/cmark-gfm/tasks/main.yml b/ansible/roles/cmark-gfm/tasks/main.yml new file mode 100644 index 0000000..3111153 --- /dev/null +++ b/ansible/roles/cmark-gfm/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Check if cmark-gfm is installed + stat: + path: /usr/local/lib/libcmark-gfm.so + register: cmark_gfm_library + +- name: Download and install cmark-gfm + when: not cmark_gfm_library.stat.exists + block: + - name: Download cmark-gfm from GitHub + get_url: + dest: /tmp/cmark-gfm.tar.gz + url: https://github.com/github/cmark-gfm/archive/0.29.0.gfm.0.tar.gz + checksum: sha256:6a94aeaa59a583fadcbf28de81dea8641b3f56d935dda5b2447a3c8df6c95fea + + - name: Create temp directory to extract cmark-gfm to + file: + path: /tmp/cmark-gfm + state: directory + + - name: Extract cmark-gfm + unarchive: + remote_src: true + src: /tmp/cmark-gfm.tar.gz + dest: /tmp/cmark-gfm + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for cmark-gfm + apt: + name: + - build-essential + - cmake + + - name: Install cmark-gfm + shell: + chdir: /tmp/cmark-gfm + cmd: | + make + make install diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000..a5e4aa4 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Set time zone to UTC + community.general.timezone: + name: Etc/UTC + +- name: Create group for app user + group: + name: "{{ app_username }}" + +- name: Create app user + user: + name: "{{ app_username }}" + group: "{{ app_username }}" diff --git a/ansible/roles/consumers/defaults/main.yml b/ansible/roles/consumers/defaults/main.yml new file mode 100644 index 0000000..d0db61d --- /dev/null +++ b/ansible/roles/consumers/defaults/main.yml @@ -0,0 +1,6 @@ +--- +consumers: + - comment_user_mentions_generator + - post_processing_script_runner + - topic_interesting_activity_updater + - topic_metadata_generator diff --git a/ansible/roles/consumers/meta/main.yml b/ansible/roles/consumers/meta/main.yml new file mode 100644 index 0000000..0a45cb3 --- /dev/null +++ b/ansible/roles/consumers/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: python + - role: redis diff --git a/ansible/roles/consumers/tasks/main.yml b/ansible/roles/consumers/tasks/main.yml new file mode 100644 index 0000000..ded3628 --- /dev/null +++ b/ansible/roles/consumers/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: Set up service files for background consumers + template: + src: "consumer.service.jinja2" + dest: /etc/systemd/system/consumer-{{ item }}.service + owner: root + group: root + mode: 0644 + loop: "{{ consumers }}" + +- name: Start and enable all consumer services + service: + name: consumer-{{ item }} + state: started + enabled: true + loop: "{{ consumers }}" diff --git a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 b/ansible/roles/consumers/templates/consumer.service.jinja2 similarity index 50% rename from salt/salt/consumers/topic_youtube_scraper.service.jinja2 rename to ansible/roles/consumers/templates/consumer.service.jinja2 index 8623e5e..4e74337 100644 --- a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 +++ b/ansible/roles/consumers/templates/consumer.service.jinja2 @@ -1,6 +1,5 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] -Description=Topic Youtube Scraper (Queue Consumer) +Description={{ item.replace("_", " ").title() }} (Queue Consumer) Requires=redis.service After=redis.service PartOf=redis.service @@ -9,8 +8,8 @@ PartOf=redis.service User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_youtube_scraper.py +Environment="INI_FILE={{ app_dir }}/{{ ini_file }}" +ExecStart={{ bin_dir }}/python {{ item }}.py Restart=always RestartSec=5 diff --git a/ansible/roles/cronjobs/tasks/main.yml b/ansible/roles/cronjobs/tasks/main.yml new file mode 100644 index 0000000..73c58df --- /dev/null +++ b/ansible/roles/cronjobs/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Add cronjob for lifting expired temporary bans + cron: + name: lift_expired_temporary_bans + job: "{{ bin_dir }}/python -c \"from scripts.lift_expired_temporary_bans import lift_expired_temporary_bans; lift_expired_temporary_bans('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 1 + +- name: Add cronjob for closing voting on old posts + cron: + name: close_voting_on_old_posts + job: "{{ bin_dir }}/python -c \"from scripts.close_voting_on_old_posts import close_voting_on_old_posts; close_voting_on_old_posts('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 3 + +- name: Add cronjob for cleaning up private data + cron: + name: clean_private_data + job: "{{ bin_dir }}/python -c \"from scripts.clean_private_data import clean_all_data; clean_all_data('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: 4 + minute: 10 + +- name: Add cronjob for generating yesterday's group stats + cron: + name: generate_group_stats_for_yesterday + job: "{{ bin_dir }}/python -c \"from scripts.generate_group_stats_for_yesterday import generate_stats; generate_stats('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: 0 + minute: 10 + +- name: Add cronjob for generating site-icons CSS file + cron: + name: generate_site_icons_css + job: "{{ bin_dir }}/python -c \"from scripts.generate_site_icons_css import generate_css; generate_css()\"" + user: "{{ app_username }}" + minute: "*/5" + +- name: Add cronjob for posting scheduled topics + cron: + name: post_scheduled_topics + job: "{{ bin_dir }}/python -c \"from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + +- name: Add cronjob for updating all groups' lists of common topic tags + cron: + name: update_groups_common_topic_tags + job: "{{ bin_dir }}/python -c \"from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 0 diff --git a/ansible/roles/development/files/ipython_config.py b/ansible/roles/development/files/ipython_config.py new file mode 100644 index 0000000..bcad1ad --- /dev/null +++ b/ansible/roles/development/files/ipython_config.py @@ -0,0 +1,2 @@ +c.InteractiveShellApp.extensions = ['autoreload'] +c.InteractiveShellApp.exec_lines = ['%autoreload 2'] diff --git a/ansible/roles/development/tasks/main.yml b/ansible/roles/development/tasks/main.yml new file mode 100644 index 0000000..4704116 --- /dev/null +++ b/ansible/roles/development/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Create IPython profile + become_user: "{{ app_username }}" + command: + cmd: "{{ bin_dir }}/ipython profile create" + creates: /home/{{ app_username }}/.ipython/profile_default + +- name: Create IPython config file + copy: + src: ipython_config.py + dest: /home/{{ app_username }}/.ipython/profile_default/ipython_config.py + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0744 + +- name: Automatically activate venv on login and in new shells + lineinfile: + path: /home/{{ app_username }}/.bashrc + line: source activate + owner: "{{ app_username }}" + group: "{{ app_username }}" + +- name: Add invoke's tab-completion script to support completing invoke task names + lineinfile: + path: /home/{{ app_username }}/.bashrc + line: source <(invoke --print-completion-script bash) diff --git a/salt/salt/gunicorn/gunicorn_reloader.path b/ansible/roles/gunicorn/files/gunicorn_reloader.path similarity index 100% rename from salt/salt/gunicorn/gunicorn_reloader.path rename to ansible/roles/gunicorn/files/gunicorn_reloader.path diff --git a/salt/salt/gunicorn/gunicorn_reloader.service b/ansible/roles/gunicorn/files/gunicorn_reloader.service similarity index 100% rename from salt/salt/gunicorn/gunicorn_reloader.service rename to ansible/roles/gunicorn/files/gunicorn_reloader.service diff --git a/ansible/roles/gunicorn/meta/main.yml b/ansible/roles/gunicorn/meta/main.yml new file mode 100644 index 0000000..f7230ad --- /dev/null +++ b/ansible/roles/gunicorn/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: python diff --git a/ansible/roles/gunicorn/tasks/main.yml b/ansible/roles/gunicorn/tasks/main.yml new file mode 100644 index 0000000..d541fde --- /dev/null +++ b/ansible/roles/gunicorn/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Create gunicorn service file + template: + src: gunicorn.service.jinja2 + dest: /etc/systemd/system/gunicorn.service + owner: root + group: root + mode: 0644 + +- name: Create gunicorn socket file + template: + src: gunicorn.socket.jinja2 + dest: /etc/systemd/system/gunicorn.socket + owner: root + group: root + mode: 0644 + +- name: Create gunicorn tmpfiles.d configuration file + template: + src: gunicorn.conf.jinja2 + dest: /usr/lib/tmpfiles.d/gunicorn.conf + owner: root + group: root + mode: 0644 + +- name: Start and enable gunicorn.socket service + service: + name: gunicorn.socket + state: started + enabled: true + +# Set up the gunicorn_reloader service, which reloads gunicorn whenever certain files +# are changed (such as static files, to update the cache-busting strings) +- name: Create gunicorn_reloader service file + copy: + src: gunicorn_reloader.service + dest: /etc/systemd/system/gunicorn_reloader.service + owner: root + group: root + mode: 0644 + +- name: Create gunicorn_reloader path-monitoring file + copy: + src: gunicorn_reloader.path + dest: /etc/systemd/system/gunicorn_reloader.path + owner: root + group: root + mode: 0644 + +- name: Start and enable gunicorn_reloader path-monitoring service + service: + name: gunicorn_reloader.path + state: started + enabled: true diff --git a/salt/salt/gunicorn/gunicorn.conf.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 similarity index 56% rename from salt/salt/gunicorn/gunicorn.conf.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 index 7554bc1..e1c3af6 100644 --- a/salt/salt/gunicorn/gunicorn.conf.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 @@ -1,2 +1 @@ -{% from 'common.jinja2' import app_username %} d /run/gunicorn 0755 {{ app_username }} {{ app_username }} - diff --git a/salt/salt/gunicorn/gunicorn.service.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.service.jinja2 similarity index 61% rename from salt/salt/gunicorn/gunicorn.service.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.service.jinja2 index 3fd06f5..fbf9912 100644 --- a/salt/salt/gunicorn/gunicorn.service.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=gunicorn daemon Requires=gunicorn.socket @@ -10,7 +9,7 @@ User={{ app_username }} Group={{ app_username }} RuntimeDirectory=gunicorn WorkingDirectory={{ app_dir }} -ExecStart={{ bin_dir }}/gunicorn --paste {{ pillar['ini_file'] }} --config {{ app_dir }}/gunicorn_config.py --bind unix:/run/gunicorn/socket --pid /run/gunicorn/pid {{ pillar['gunicorn_args'] }} +ExecStart={{ bin_dir }}/gunicorn --paste {{ ini_file }} --config {{ app_dir }}/gunicorn_config.py --bind unix:/run/gunicorn/socket --pid /run/gunicorn/pid {{ gunicorn_args }} ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true diff --git a/salt/salt/gunicorn/gunicorn.socket.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 similarity index 80% rename from salt/salt/gunicorn/gunicorn.socket.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 index a81da97..3773779 100644 --- a/salt/salt/gunicorn/gunicorn.socket.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_username -%} [Unit] Description=gunicorn socket PartOf=gunicorn.service diff --git a/ansible/roles/ipv6_networking/defaults/main.yml b/ansible/roles/ipv6_networking/defaults/main.yml new file mode 100644 index 0000000..99539e9 --- /dev/null +++ b/ansible/roles/ipv6_networking/defaults/main.yml @@ -0,0 +1,2 @@ +--- +ipv6_device: eth0 diff --git a/ansible/roles/ipv6_networking/tasks/main.yml b/ansible/roles/ipv6_networking/tasks/main.yml new file mode 100644 index 0000000..8a8d847 --- /dev/null +++ b/ansible/roles/ipv6_networking/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Enable IPv6 networking + blockinfile: + path: /etc/network/interfaces + block: | + iface {{ ipv6_device }} inet6 static + address {{ ipv6_address }} + netmask 64 + + post-up sleep 5; /sbin/ip -family inet6 route add {{ ipv6_gateway }} dev {{ ipv6_device }} + post-up sleep 5; /sbin/ip -family inet6 route add default via {{ ipv6_gateway }} + pre-down /sbin/ip -family inet6 route del default via {{ ipv6_gateway }} + pre-down /sbin/ip -family inet6 route del {{ ipv6_gateway }} dev {{ ipv6_device }} + +# apt seems to hang a lot when using IPv6 +- name: Force apt not to use IPv6 + lineinfile: + path: /etc/apt/apt.conf.d/99force-ipv4 + line: Acquire::ForceIPv4 "true"; + create: true + owner: root + group: root + mode: 0644 diff --git a/ansible/roles/java/tasks/main.yml b/ansible/roles/java/tasks/main.yml new file mode 100644 index 0000000..dfa1e0b --- /dev/null +++ b/ansible/roles/java/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Install OpenJDK Java runtime + apt: + name: openjdk-11-jre diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml new file mode 100644 index 0000000..eacc87f --- /dev/null +++ b/ansible/roles/nginx/defaults/main.yml @@ -0,0 +1,6 @@ +--- +nginx_enable_sendfile: true +nginx_worker_processes: auto + +ssl_cert_path: /etc/letsencrypt/live/{{ site_hostname }}/fullchain.pem +ssl_private_key_path: /etc/letsencrypt/live/{{ site_hostname }}/privkey.pem diff --git a/salt/salt/nginx/logrotate b/ansible/roles/nginx/files/logrotate similarity index 86% rename from salt/salt/nginx/logrotate rename to ansible/roles/nginx/files/logrotate index 9a4fc89..a089752 100644 --- a/salt/salt/nginx/logrotate +++ b/ansible/roles/nginx/files/logrotate @@ -1,3 +1,4 @@ +# rotate nginx log files daily and delete after 30 days /var/log/nginx/*.log { daily missingok diff --git a/ansible/roles/nginx/handlers/main.yml b/ansible/roles/nginx/handlers/main.yml new file mode 100644 index 0000000..eb0e7f1 --- /dev/null +++ b/ansible/roles/nginx/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml new file mode 100644 index 0000000..467449d --- /dev/null +++ b/ansible/roles/nginx/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: Add APT key for nginx repository + apt_key: + url: https://nginx.org/keys/nginx_signing.key + +- name: Add nginx APT repository + apt_repository: + repo: deb http://nginx.org/packages/debian/ buster nginx + +- name: Install nginx + apt: + name: nginx + +- name: Start and enable nginx service + service: + name: nginx + state: started + enabled: true + +- name: Create nginx.conf file + template: + src: nginx.conf.jinja2 + dest: /etc/nginx/nginx.conf + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Create sites-available directory + file: + path: /etc/nginx/sites-available + state: directory + owner: root + group: root + mode: 0755 + +- name: Create sites-enabled directory + file: + path: /etc/nginx/sites-enabled + state: directory + owner: root + group: root + mode: 0744 + +- name: Add logrotate config + copy: + src: logrotate + dest: /etc/logrotate.d/nginx + owner: root + group: root + mode: 0644 diff --git a/salt/salt/nginx/nginx.conf.jinja2 b/ansible/roles/nginx/templates/nginx.conf.jinja2 similarity index 89% rename from salt/salt/nginx/nginx.conf.jinja2 rename to ansible/roles/nginx/templates/nginx.conf.jinja2 index 493fd01..af1efc5 100644 --- a/salt/salt/nginx/nginx.conf.jinja2 +++ b/ansible/roles/nginx/templates/nginx.conf.jinja2 @@ -1,5 +1,5 @@ user nginx; -worker_processes {{ pillar['nginx_worker_processes'] }}; +worker_processes {{ nginx_worker_processes }}; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; @@ -22,11 +22,10 @@ http { access_log /var/log/nginx/access.log main; - {% if grains['id'] == 'dev' %} - # have to disable sendfile for vagrant due to a virtualbox bug - sendfile off; - {% else %} + {% if nginx_enable_sendfile %} sendfile on; + {% else %} + sendfile off; {% endif %} # define a rate-limiting zone to use, and return HTTP 429 if exceeded @@ -77,8 +76,8 @@ http { text/x-component text/x-cross-domain-policy; - ssl_certificate {{ pillar['ssl_cert_path'] }}; - ssl_certificate_key {{ pillar['ssl_private_key_path'] }}; + ssl_certificate {{ ssl_cert_path }}; + ssl_certificate_key {{ ssl_private_key_path }}; ssl_session_timeout 1d; ssl_session_cache shared:SSL:50m; ssl_session_tickets off; diff --git a/ansible/roles/nginx_prod_config/defaults/main.yml b/ansible/roles/nginx_prod_config/defaults/main.yml new file mode 100644 index 0000000..fbb06da --- /dev/null +++ b/ansible/roles/nginx_prod_config/defaults/main.yml @@ -0,0 +1,3 @@ +--- +hsts_max_age: 63072000 +static_sites_dir: /opt/tildes-static-sites diff --git a/ansible/roles/nginx_prod_config/meta/main.yml b/ansible/roles/nginx_prod_config/meta/main.yml new file mode 100644 index 0000000..80b68ef --- /dev/null +++ b/ansible/roles/nginx_prod_config/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: nginx + - role: gunicorn diff --git a/ansible/roles/nginx_prod_config/tasks/main.yml b/ansible/roles/nginx_prod_config/tasks/main.yml new file mode 100644 index 0000000..bfc3b0e --- /dev/null +++ b/ansible/roles/nginx_prod_config/tasks/main.yml @@ -0,0 +1,38 @@ +--- +- name: Add shortener config file + template: + src: tildes-shortener.conf.jinja2 + dest: /etc/nginx/sites-available/tildes-shortener.conf + owner: root + group: root + mode: 0644 + +- name: Enable shortener in nginx + file: + path: /etc/nginx/sites-enabled/tildes-shortener.conf + src: /etc/nginx/sites-available/tildes-shortener.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Add static sites config file + template: + src: tildes-static-sites.conf.jinja2 + dest: /etc/nginx/sites-available/tildes-static-sites.conf + owner: root + group: root + mode: 0644 + +- name: Enable static sites in nginx + file: + path: /etc/nginx/sites-enabled/tildes-static-sites.conf + src: /etc/nginx/sites-available/tildes-static-sites.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx diff --git a/salt/salt/nginx/tildes-shortener.conf.jinja2 b/ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 similarity index 84% rename from salt/salt/nginx/tildes-shortener.conf.jinja2 rename to ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 index 019f3fd..9ef29d1 100644 --- a/salt/salt/nginx/tildes-shortener.conf.jinja2 +++ b/ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir -%} server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -7,7 +6,7 @@ server { keepalive_timeout 5; - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; # Are these security headers unnecessary when we're just redirecting? add_header X-Content-Type-Options "nosniff" always; diff --git a/salt/salt/nginx/tildes-static-sites.conf.jinja2 b/ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 similarity index 91% rename from salt/salt/nginx/tildes-static-sites.conf.jinja2 rename to ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 index 869bcf6..291dc71 100644 --- a/salt/salt/nginx/tildes-static-sites.conf.jinja2 +++ b/ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 @@ -1,11 +1,9 @@ -{% from 'common.jinja2' import static_sites_dir -%} - {% for subdomain in ('blog', 'docs') %} server { listen 443 ssl http2; listen [::]:443 ssl http2; - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; add_header X-Content-Type-Options "nosniff" always; add_header X-Frame-Options "SAMEORIGIN" always; diff --git a/ansible/roles/nginx_site_config/defaults/main.yml b/ansible/roles/nginx_site_config/defaults/main.yml new file mode 100644 index 0000000..7a51abd --- /dev/null +++ b/ansible/roles/nginx_site_config/defaults/main.yml @@ -0,0 +1,11 @@ +--- +nginx_enable_hsts: true +nginx_enable_csp: true +nginx_enable_ratelimiting: true +nginx_redirect_www: true + +prometheus_ips: + - 127.0.0.1 + - ::1 + +hsts_max_age: 63072000 diff --git a/ansible/roles/nginx_site_config/meta/main.yml b/ansible/roles/nginx_site_config/meta/main.yml new file mode 100644 index 0000000..80b68ef --- /dev/null +++ b/ansible/roles/nginx_site_config/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: nginx + - role: gunicorn diff --git a/ansible/roles/nginx_site_config/tasks/main.yml b/ansible/roles/nginx_site_config/tasks/main.yml new file mode 100644 index 0000000..57bcd8e --- /dev/null +++ b/ansible/roles/nginx_site_config/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Add site config file + template: + src: tildes.conf.jinja2 + dest: /etc/nginx/sites-available/tildes.conf + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Enable site in nginx + file: + path: /etc/nginx/sites-enabled/tildes.conf + src: /etc/nginx/sites-available/tildes.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx diff --git a/salt/salt/nginx/tildes.conf.jinja2 b/ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 similarity index 87% rename from salt/salt/nginx/tildes.conf.jinja2 rename to ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 index df7749e..92795cf 100644 --- a/salt/salt/nginx/tildes.conf.jinja2 +++ b/ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir -%} upstream app_server { server unix:/run/gunicorn/socket fail_timeout=0; } @@ -35,9 +34,11 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; - {% if grains['id'] != 'dev' %} - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + {% if nginx_enable_hsts %} + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; + {% endif %} + {% if nginx_enable_csp %} # Content Security Policy: # - "img-src data:" is needed for Spectre.css icons # - "https://js.stripe.com" in script-src and frame-src is needed for Stripe @@ -49,7 +50,7 @@ server { add_header X-Xss-Protection "1; mode=block" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; - server_name {{ pillar['site_hostname'] }}; + server_name {{ site_hostname }}; keepalive_timeout 5; @@ -57,9 +58,9 @@ server { # Block access to /metrics except from Prometheus server(s) location /metrics { - {% for ip in pillar['prometheus_ips'] %} + {% for ip in prometheus_ips -%} allow {{ ip }}; - {% endfor %} + {% endfor -%} deny all; # try_files is unnecessary here, but I don't know the "proper" way @@ -76,7 +77,7 @@ server { } location @proxy_to_app { - {% if grains["id"] == "prod" %} + {% if nginx_enable_ratelimiting %} # apply rate-limiting, allowing a burst above the limit limit_req zone=tildes_app burst=5 nodelay; {% endif -%} @@ -96,14 +97,14 @@ server { } } -{% if grains["id"] == "prod" %} +{% if nginx_redirect_www %} # redirect www. to base domain server { listen 80; listen 443 ssl http2; listen [::]:443 ssl http2; - server_name www.tildes.net; - return 301 https://tildes.net$request_uri; + server_name www.{{ site_hostname }}; + return 301 https://{{ site_hostname }}$request_uri; } {% endif %} diff --git a/ansible/roles/nodejs/tasks/main.yml b/ansible/roles/nodejs/tasks/main.yml new file mode 100644 index 0000000..2138af3 --- /dev/null +++ b/ansible/roles/nodejs/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Add APT key for NodeSource Node.js repository + apt_key: + url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key + +- name: Add NodeSource Node.js APT repository + apt_repository: + repo: deb https://deb.nodesource.com/node_14.x buster main + +- name: Install Node.js + apt: + name: nodejs + +- name: Install npm packages defined in package.json + become_user: "{{ app_username }}" + community.general.npm: + path: "{{ app_dir }}" + # --no-bin-links option is needed to prevent npm from creating symlinks in the .bin + # directory, which doesn't work inside Vagrant on Windows + no_bin_links: true diff --git a/ansible/roles/pgbouncer/handlers/main.yml b/ansible/roles/pgbouncer/handlers/main.yml new file mode 100644 index 0000000..0d14811 --- /dev/null +++ b/ansible/roles/pgbouncer/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload pgbouncer + service: + name: pgbouncer + state: reloaded diff --git a/ansible/roles/pgbouncer/meta/main.yml b/ansible/roles/pgbouncer/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/pgbouncer/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/pgbouncer/tasks/main.yml b/ansible/roles/pgbouncer/tasks/main.yml new file mode 100644 index 0000000..ce187b6 --- /dev/null +++ b/ansible/roles/pgbouncer/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Install pgbouncer + apt: + name: pgbouncer + +- name: Add pgbouncer.ini + template: + src: pgbouncer.ini.jinja2 + dest: /etc/pgbouncer/pgbouncer.ini + owner: postgres + group: postgres + mode: 0640 + notify: + - Reload pgbouncer + +- name: Add user to pgbouncer userlist + lineinfile: + path: /etc/pgbouncer/userlist.txt + line: '"tildes" ""' + create: true + owner: postgres + group: postgres + mode: 0640 + notify: + - Reload pgbouncer + +- name: Start and enable pgbouncer service + service: + name: pgbouncer + state: started + enabled: true diff --git a/salt/salt/postgresql/pgbouncer.ini.jinja2 b/ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 similarity index 64% rename from salt/salt/postgresql/pgbouncer.ini.jinja2 rename to ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 index 9b89c8b..da7b5fe 100644 --- a/salt/salt/postgresql/pgbouncer.ini.jinja2 +++ b/ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 @@ -1,7 +1,4 @@ [databases] -{% for line in accumulator['pgbouncer_db_lines'] -%} -{{ line }} -{% endfor %} [pgbouncer] logfile = /var/log/postgresql/pgbouncer.log @@ -16,6 +13,6 @@ unix_socket_dir = /var/run/postgresql auth_type = hba auth_file = /etc/pgbouncer/userlist.txt -auth_hba_file = /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf +auth_hba_file = /etc/postgresql/{{ postgresql_version }}/main/pg_hba.conf pool_mode = transaction diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml new file mode 100644 index 0000000..6420521 --- /dev/null +++ b/ansible/roles/postgresql/defaults/main.yml @@ -0,0 +1,12 @@ +--- +postgresql_version: 12 + +# Users of this role can define postgresql_settings, which will be merged with +# this base _postgresql_settings +_postgresql_settings: + lock_timeout: 5000 + statement_timeout: 5000 + idle_in_transaction_session_timeout: 600000 + timezone: "'UTC'" + shared_preload_libraries: "'pg_stat_statements'" +postgresql_settings: {} diff --git a/ansible/roles/postgresql/handlers/main.yml b/ansible/roles/postgresql/handlers/main.yml new file mode 100644 index 0000000..4a6d976 --- /dev/null +++ b/ansible/roles/postgresql/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: Restart postgresql + service: + name: postgresql + state: restarted + +- name: Reload postgresql + service: + name: postgresql + state: reloaded diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml new file mode 100644 index 0000000..f925e67 --- /dev/null +++ b/ansible/roles/postgresql/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Add APT key for PostgreSQL repository + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + +- name: Add PostgreSQL APT repository + apt_repository: + repo: deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main + +- name: Install PostgreSQL + apt: + name: postgresql-{{ postgresql_version }} + +- name: Start and enable PostgreSQL service + service: + name: postgresql + state: started + enabled: true + +- name: Set configuration options in postgresql.conf + lineinfile: + path: /etc/postgresql/{{ postgresql_version }}/main/postgresql.conf + regexp: "^#?{{ item.key }} ?=" + line: "{{ item.key }} = {{ item.value }}" + loop: "{{ _postgresql_settings | combine(postgresql_settings) | dict2items }}" + notify: + - Restart postgresql diff --git a/ansible/roles/postgresql_plpython/meta/main.yml b/ansible/roles/postgresql_plpython/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/postgresql_plpython/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/postgresql_plpython/tasks/main.yml b/ansible/roles/postgresql_plpython/tasks/main.yml new file mode 100644 index 0000000..c6fab8d --- /dev/null +++ b/ansible/roles/postgresql_plpython/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Install PL/Python3 procedural language for PostgreSQL + apt: + name: postgresql-plpython3-{{ postgresql_version }} + +- name: Set PYTHONPATH env var for PostgreSQL + lineinfile: + path: /etc/postgresql/{{ postgresql_version }}/main/environment + regexp: "^PYTHONPATH=" + line: "PYTHONPATH='{{ venv_dir }}/lib/python{{ python_version }}/site-packages:{{ app_dir }}'" + notify: + - Restart postgresql diff --git a/ansible/roles/postgresql_redis_bridge/meta/main.yml b/ansible/roles/postgresql_redis_bridge/meta/main.yml new file mode 100644 index 0000000..edeef00 --- /dev/null +++ b/ansible/roles/postgresql_redis_bridge/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: postgresql + - role: redis diff --git a/ansible/roles/postgresql_redis_bridge/tasks/main.yml b/ansible/roles/postgresql_redis_bridge/tasks/main.yml new file mode 100644 index 0000000..d517275 --- /dev/null +++ b/ansible/roles/postgresql_redis_bridge/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Create postgresql_redis_bridge service file + template: + src: postgresql_redis_bridge.service.jinja2 + dest: /etc/systemd/system/postgresql_redis_bridge.service + owner: root + group: root + mode: 0644 + +- name: Start and enable postgresql_redis_bridge service + service: + name: postgresql_redis_bridge + state: started + enabled: true diff --git a/salt/salt/postgresql_redis_bridge.service.jinja2 b/ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 similarity index 73% rename from salt/salt/postgresql_redis_bridge.service.jinja2 rename to ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 index be2a5f3..df485cb 100644 --- a/salt/salt/postgresql_redis_bridge.service.jinja2 +++ b/ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=postgresql_redis_bridge - convert NOTIFY to Redis streams Requires=redis.service @@ -9,7 +8,7 @@ PartOf=redis.service User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }}/scripts -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" +Environment="INI_FILE={{ app_dir }}/{{ ini_file }}" ExecStart={{ bin_dir }}/python postgresql_redis_bridge.py Restart=always RestartSec=5 diff --git a/ansible/roles/postgresql_tildes_dbs/defaults/main.yml b/ansible/roles/postgresql_tildes_dbs/defaults/main.yml new file mode 100644 index 0000000..52acffa --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/defaults/main.yml @@ -0,0 +1,6 @@ +--- +postgresql_tildes_databases: + - tildes +postgresql_tildes_user_flags: "" + +tildes_database_insert_dev_data: false diff --git a/ansible/roles/postgresql_tildes_dbs/meta/main.yml b/ansible/roles/postgresql_tildes_dbs/meta/main.yml new file mode 100644 index 0000000..008e938 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: postgresql + - role: pgbouncer + + # needed to be able to run the db init scripts + - role: python + - role: cmark-gfm diff --git a/ansible/roles/postgresql_tildes_dbs/tasks/database.yml b/ansible/roles/postgresql_tildes_dbs/tasks/database.yml new file mode 100644 index 0000000..4be87f5 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/tasks/database.yml @@ -0,0 +1,38 @@ +--- +- name: Create database and enable access and all necessary extensions + become_user: postgres + block: + - name: Create database + community.postgresql.postgresql_db: + name: "{{ item }}" + owner: tildes + + - name: Enable extensions + community.postgresql.postgresql_ext: + name: "{{ extension }}" + db: "{{ item }}" + loop: "{{ extensions }}" + # since the "outer" loop by include_tasks is already using the `item` variable, + # this will use `extension` for the loop inside here + loop_control: + loop_var: extension + + - name: Add database to pg_hba.conf + community.postgresql.postgresql_pg_hba: + dest: /etc/postgresql/{{ postgresql_version }}/main/pg_hba.conf + databases: "{{ item }}" + users: tildes + contype: local + method: trust + notify: + - Reload postgresql + + - name: Add database to pgbouncer.ini + become_user: root + lineinfile: + path: /etc/pgbouncer/pgbouncer.ini + line: "{{ item }} =" + insertafter: "^\\[databases\\]$" + firstmatch: true + notify: + - Reload pgbouncer diff --git a/ansible/roles/postgresql_tildes_dbs/tasks/main.yml b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml new file mode 100644 index 0000000..3911715 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml @@ -0,0 +1,62 @@ +--- +- name: Install requirements for building psycopg2 (needed by Ansible) + apt: + name: + - gcc + - libpq-dev + - python3-dev + +- name: Install packages needed by Ansible community plugins + pip: + executable: pip3 + name: + - ipaddress + - psycopg2 + +- name: Create tildes user + become_user: postgres + community.postgresql.postgresql_user: + name: tildes + role_attr_flags: "{{ postgresql_tildes_user_flags }}" + +# This is a bit of a hack to effectively enable looping over a block of tasks +- name: Set up site database (and test database in dev version) + include_tasks: database.yml + loop: "{{ postgresql_tildes_databases }}" + vars: + extensions: + - citext + - ltree + - intarray + - pg_stat_statements + - pg_trgm + - plpython3u + register: database_changes + +# Since handlers don't run until the end of the entire playbook, we need to run them +# manually at this point in case postgresql or pgbouncer need to be reloaded +- name: Trigger handlers to run manually for postgresql/pgbouncer updates + meta: flush_handlers + +- name: Check if the database has already been initialized (will fail if not) + become_user: postgres + community.postgresql.postgresql_query: + db: tildes + query: select user_id from users; + ignore_errors: true + register: users_query + +- name: Initialize the database + become_user: postgres + command: + cmd: "{{ bin_dir }}/python -c \"from scripts.initialize_db import initialize_db; initialize_db('{{ app_dir }}/{{ ini_file }}')\"" + chdir: "{{ app_dir }}" + when: users_query is failed + register: initialize_db + +- name: Insert dev data into database + become_user: "{{ app_username }}" + command: + cmd: "{{ bin_dir }}/python -c \"from scripts.initialize_db import insert_dev_data; insert_dev_data('{{ app_dir }}/{{ ini_file }}')\"" + chdir: "{{ app_dir }}" + when: tildes_database_insert_dev_data and initialize_db is changed diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml new file mode 100644 index 0000000..0a963aa --- /dev/null +++ b/ansible/roles/prometheus/defaults/main.yml @@ -0,0 +1,8 @@ +--- +prometheus_version: 2.0.0 + +prometheus_consumer_scrape_targets: + comment_user_mentions_generator: 25010 + post_processing_script_runner: 25016 + topic_interesting_activity_updater: 25013 + topic_metadata_generator: 25014 diff --git a/salt/salt/prometheus/prometheus.service b/ansible/roles/prometheus/files/prometheus.service similarity index 100% rename from salt/salt/prometheus/prometheus.service rename to ansible/roles/prometheus/files/prometheus.service diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml new file mode 100644 index 0000000..2f5a4b8 --- /dev/null +++ b/ansible/roles/prometheus/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus + service: + name: prometheus + state: restarted diff --git a/ansible/roles/prometheus/tasks/main.yml b/ansible/roles/prometheus/tasks/main.yml new file mode 100644 index 0000000..292156d --- /dev/null +++ b/ansible/roles/prometheus/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Check if Prometheus is installed + stat: + path: /opt/prometheus/prometheus + register: prometheus_binary + +- name: Download and install Prometheus + when: not prometheus_binary.stat.exists + block: + - name: Download Prometheus from GitHub + get_url: + dest: /tmp/prometheus-{{ prometheus_version }}.tar.gz + url: https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz + checksum: sha256:e12917b25b32980daee0e9cf879d9ec197e2893924bd1574604eb0f550034d46 + + - name: Create Prometheus directory + file: + path: /opt/prometheus + state: directory + owner: prometheus + group: prometheus + mode: 0755 + + - name: Extract Prometheus + unarchive: + remote_src: true + src: /tmp/prometheus-{{ prometheus_version }}.tar.gz + dest: /opt/prometheus + owner: prometheus + group: prometheus + extra_opts: + - --strip-components=1 + +- name: Create Prometheus service file + copy: + src: prometheus.service + dest: /etc/systemd/system/prometheus.service + owner: root + group: root + mode: 0644 + +- name: Add Prometheus config file + template: + src: prometheus.yml.jinja2 + dest: /opt/prometheus/prometheus.yml + owner: prometheus + group: prometheus + mode: 0644 + notify: + - Restart prometheus + +- name: Start and enable prometheus service + service: + name: prometheus + state: started + enabled: true diff --git a/salt/salt/prometheus/prometheus.yml.jinja2 b/ansible/roles/prometheus/templates/prometheus.yml.jinja2 similarity index 53% rename from salt/salt/prometheus/prometheus.yml.jinja2 rename to ansible/roles/prometheus/templates/prometheus.yml.jinja2 index 1907a31..4fea515 100644 --- a/salt/salt/prometheus/prometheus.yml.jinja2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.jinja2 @@ -5,21 +5,21 @@ global: scrape_configs: - job_name: "node" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9100'] + - targets: ['{{ site_hostname }}:9100'] - job_name: "redis" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9121'] + - targets: ['{{ site_hostname }}:9121'] - job_name: "postgres" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9187'] + - targets: ['{{ site_hostname }}:9187'] - job_name: "tildes" scheme: https static_configs: - - targets: ['{{ pillar['site_hostname'] }}:443'] - {% if grains['id'] == 'dev' %} + - targets: ['{{ site_hostname }}:443'] + {% if site_hostname == "localhost" -%} tls_config: insecure_skip_verify: true {% endif %} @@ -31,7 +31,7 @@ scrape_configs: module: [site_ipv4] static_configs: - targets: - - https://{{ pillar['site_hostname'] }} + - https://{{ site_hostname }} relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -47,7 +47,7 @@ scrape_configs: module: [site_ipv6] static_configs: - targets: - - https://{{ pillar['site_hostname'] }} + - https://{{ site_hostname }} relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -57,28 +57,8 @@ scrape_configs: replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port # event stream consumers (background jobs) - - job_name: "consumer_comment_user_mentions_generator" + {% for name, port in prometheus_consumer_scrape_targets.items() -%} + - job_name: "consumer_{{ name }}" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25010'] - - - job_name: "consumer_topic_interesting_activity_updater" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25013'] - - - job_name: "consumer_topic_metadata_generator" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25014'] - - {% if grains["id"] != "dev" %} - - job_name: "consumer_site_icon_downloader" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25011'] - - - job_name: "consumer_topic_embedly_extractor" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25012'] - - - job_name: "consumer_topic_youtube_scraper" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25015'] - {% endif %} + - targets: ['{{ site_hostname }}:{{ port }}'] + {% endfor %} diff --git a/salt/salt/prometheus/exporters/prometheus_node_exporter.service b/ansible/roles/prometheus_node_exporter/files/prometheus_node_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_node_exporter.service rename to ansible/roles/prometheus_node_exporter/files/prometheus_node_exporter.service diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml new file mode 100644 index 0000000..e7cac53 --- /dev/null +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Download node_exporter from GitHub + get_url: + dest: /tmp/prometheus_node_exporter.tar.gz + url: https://github.com/prometheus/node_exporter/releases/download/v0.13.0/node_exporter-0.13.0.linux-amd64.tar.gz + checksum: sha256:2de5d1e51330c41588ed4c88bc531a3d2dccf6b4d7b99d5782d95cff27a3c049 + +- name: Create node_exporter directory + file: + path: /opt/prometheus_node_exporter + state: directory + owner: prometheus + group: prometheus + mode: 0755 + +- name: Extract node_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_node_exporter.tar.gz + dest: /opt/prometheus_node_exporter + owner: prometheus + group: prometheus + extra_opts: + - --strip-components=1 + creates: /opt/prometheus_node_exporter/node_exporter + +- name: Create node_exporter service file + copy: + src: prometheus_node_exporter.service + dest: /etc/systemd/system/prometheus_node_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable node_exporter service + service: + name: prometheus_node_exporter + state: started + enabled: true diff --git a/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service b/ansible/roles/prometheus_postgres_exporter/files/prometheus_postgres_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_postgres_exporter.service rename to ansible/roles/prometheus_postgres_exporter/files/prometheus_postgres_exporter.service diff --git a/salt/salt/prometheus/exporters/queries.yaml b/ansible/roles/prometheus_postgres_exporter/files/queries.yaml similarity index 100% rename from salt/salt/prometheus/exporters/queries.yaml rename to ansible/roles/prometheus_postgres_exporter/files/queries.yaml diff --git a/ansible/roles/prometheus_postgres_exporter/meta/main.yml b/ansible/roles/prometheus_postgres_exporter/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/prometheus_postgres_exporter/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/prometheus_postgres_exporter/tasks/main.yml b/ansible/roles/prometheus_postgres_exporter/tasks/main.yml new file mode 100644 index 0000000..55cd99d --- /dev/null +++ b/ansible/roles/prometheus_postgres_exporter/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Download postgres_exporter from GitHub + get_url: + dest: /tmp/prometheus_postgres_exporter.tar.gz + url: https://github.com/wrouesnel/postgres_exporter/releases/download/v0.4.7/postgres_exporter_v0.4.7_linux-amd64.tar.gz + checksum: sha256:c34d61bb4deba8efae06fd3c9979b96dae3f3c757698ce3384c80fff586c667b + +- name: Create postgres_exporter directory + file: + path: /opt/prometheus_postgres_exporter + state: directory + owner: postgres + group: postgres + mode: 0755 + +- name: Extract postgres_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_postgres_exporter.tar.gz + dest: /opt/prometheus_postgres_exporter + owner: postgres + group: postgres + extra_opts: + - --strip-components=1 + creates: /opt/prometheus_postgres_exporter/postgres_exporter + +- name: Create queries.yaml file + copy: + src: queries.yaml + dest: /opt/prometheus_postgres_exporter/queries.yaml + owner: postgres + group: postgres + mode: 0644 + +- name: Create postgres_exporter service file + copy: + src: prometheus_postgres_exporter.service + dest: /etc/systemd/system/prometheus_postgres_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable postgres_exporter service + service: + name: prometheus_postgres_exporter + state: started + enabled: true diff --git a/salt/salt/prometheus/exporters/prometheus_redis_exporter.service b/ansible/roles/prometheus_redis_exporter/files/prometheus_redis_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_redis_exporter.service rename to ansible/roles/prometheus_redis_exporter/files/prometheus_redis_exporter.service diff --git a/ansible/roles/prometheus_redis_exporter/meta/main.yml b/ansible/roles/prometheus_redis_exporter/meta/main.yml new file mode 100644 index 0000000..89a7c92 --- /dev/null +++ b/ansible/roles/prometheus_redis_exporter/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: redis diff --git a/ansible/roles/prometheus_redis_exporter/tasks/main.yml b/ansible/roles/prometheus_redis_exporter/tasks/main.yml new file mode 100644 index 0000000..bf98d6f --- /dev/null +++ b/ansible/roles/prometheus_redis_exporter/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Download redis_exporter from GitHub + get_url: + dest: /tmp/prometheus_redis_exporter.tar.gz + url: https://github.com/oliver006/redis_exporter/releases/download/v0.26.0/redis_exporter-v0.26.0.linux-amd64.tar.gz + checksum: sha256:39354c57b9d02b455c584baf46a2df6ed3d1ac190c88e3ec0fa0c23b49ccc2bb + +- name: Create redis_exporter directory + file: + path: /opt/prometheus_redis_exporter + state: directory + owner: prometheus + group: prometheus + mode: 0755 + +- name: Extract redis_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_redis_exporter.tar.gz + dest: /opt/prometheus_redis_exporter + owner: prometheus + group: prometheus + creates: /opt/prometheus_redis_exporter/redis_exporter + +- name: Create redis_exporter service file + copy: + src: prometheus_redis_exporter.service + dest: /etc/systemd/system/prometheus_redis_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable redis_exporter service + service: + name: prometheus_redis_exporter + state: started + enabled: true diff --git a/ansible/roles/pts_lbsearch/tasks/main.yml b/ansible/roles/pts_lbsearch/tasks/main.yml new file mode 100644 index 0000000..703be93 --- /dev/null +++ b/ansible/roles/pts_lbsearch/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Download pts_lbsearch code from GitHub + get_url: + dest: /tmp/pts_lbsearch.c + url: https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c + checksum: sha256:ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1 + +- name: Compile pts_lbsearch + command: + chdir: /tmp + # compilation command taken from the top of the source file + cmd: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c + creates: /usr/local/bin/pts_lbsearch diff --git a/ansible/roles/python/defaults/main.yml b/ansible/roles/python/defaults/main.yml new file mode 100644 index 0000000..420c1cc --- /dev/null +++ b/ansible/roles/python/defaults/main.yml @@ -0,0 +1,2 @@ +--- +pip_requirements_filename: requirements.txt diff --git a/ansible/roles/python/tasks/main.yml b/ansible/roles/python/tasks/main.yml new file mode 100644 index 0000000..4bf9c62 --- /dev/null +++ b/ansible/roles/python/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Check if the correct version of Python is already installed + stat: + path: /usr/local/bin/python{{ python_version }} + register: python_binary + +- name: Download and install Python + when: not python_binary.stat.exists + block: + - name: Download Python source code + get_url: + dest: /tmp/python.tar.gz + url: https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tgz + checksum: sha256:b37ac74d2cbad2590e7cd0dd2b3826c29afe89a734090a87bf8c03c45066cb65 + + - name: Create temp directory to extract Python to + file: + path: /tmp/python + state: directory + + - name: Extract Python + unarchive: + remote_src: true + src: /tmp/python.tar.gz + dest: /tmp/python + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for Python + apt: + name: + - make + - build-essential + - libssl-dev + - zlib1g-dev + - libbz2-dev + - libreadline-dev + - libsqlite3-dev + - wget + - curl + - llvm + - libncurses5-dev + - libncursesw5-dev + - xz-utils + - tk-dev + + - name: Build and install Python (this can take a long time) + shell: + chdir: /tmp/python + cmd: | + ./configure --enable-optimizations --with-ensurepip=install + make + make altinstall + +- name: Create dir for venvs + file: + path: /opt/venvs + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0755 + +- name: Install packages needed for compiling psycopg2 + apt: + name: + - gcc + - libpq-dev + +- name: Create venv + become_user: "{{ app_username }}" + command: + cmd: python{{ python_version }} -m venv {{ venv_dir }} + creates: "{{ venv_dir }}" + +- name: Check installed packages in venv + command: + cmd: "{{ bin_dir }}/pip freeze" + register: pip_freeze + changed_when: false + +- name: Install Python packages into venv + become_user: "{{ app_username }}" + pip: + executable: "{{ bin_dir }}/pip" + requirements: "{{ app_dir }}/{{ pip_requirements_filename }}" + +- name: Install Tildes into venv as editable + become_user: "{{ app_username }}" + pip: + executable: "{{ bin_dir }}/pip" + name: "{{ app_dir }}" + editable: true + when: "'-e '+app_dir not in pip_freeze.stdout_lines" diff --git a/ansible/roles/redis/defaults/main.yml b/ansible/roles/redis/defaults/main.yml new file mode 100644 index 0000000..659e9a2 --- /dev/null +++ b/ansible/roles/redis/defaults/main.yml @@ -0,0 +1,2 @@ +--- +redis_version: 5.0.7 diff --git a/salt/salt/redis/redis.service b/ansible/roles/redis/files/redis.service similarity index 100% rename from salt/salt/redis/redis.service rename to ansible/roles/redis/files/redis.service diff --git a/salt/salt/redis/transparent_hugepage.service b/ansible/roles/redis/files/transparent_hugepage.service similarity index 100% rename from salt/salt/redis/transparent_hugepage.service rename to ansible/roles/redis/files/transparent_hugepage.service diff --git a/ansible/roles/redis/handlers/main.yml b/ansible/roles/redis/handlers/main.yml new file mode 100644 index 0000000..1ba7b3d --- /dev/null +++ b/ansible/roles/redis/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart redis + service: + name: redis + state: restarted diff --git a/ansible/roles/redis/tasks/main.yml b/ansible/roles/redis/tasks/main.yml new file mode 100644 index 0000000..9a76822 --- /dev/null +++ b/ansible/roles/redis/tasks/main.yml @@ -0,0 +1,131 @@ +--- +- name: Check if Redis is installed + stat: + path: /usr/local/bin/redis-server + register: redis_server + +- name: Download and install Redis + when: not redis_server.stat.exists + block: + - name: Download Redis from GitHub + get_url: + dest: /tmp/redis-{{ redis_version }}.tar.gz + url: https://github.com/antirez/redis/archive/{{ redis_version }}.tar.gz + checksum: sha256:2761422599f8969559e66797cd7f606c16e907bf82d962345a7d366c5d1278df + + - name: Create temp directory to extract Redis to + file: + path: /tmp/redis-{{ redis_version }} + state: directory + + - name: Extract Redis + unarchive: + remote_src: true + src: /tmp/redis-{{ redis_version }}.tar.gz + dest: /tmp/redis-{{ redis_version }} + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for Redis + apt: + name: build-essential + + - name: Install Redis + shell: + chdir: /tmp/redis-{{ redis_version }} + cmd: | + make + make install + +- name: Create group for redis user + group: + name: redis + +- name: Create redis user + user: + name: redis + group: redis + create_home: false + +- name: Create /run/redis + file: + path: /run/redis + state: directory + owner: redis + group: redis + mode: 0755 + +- name: Create /var/lib/redis + file: + path: /var/lib/redis + state: directory + owner: redis + group: redis + mode: 0700 + +- name: Create /var/log/redis + file: + path: /var/log/redis + state: directory + owner: redis + group: redis + mode: 0744 + +- name: Create Redis configuration file + template: + src: redis.conf.jinja2 + dest: /etc/redis.conf + owner: redis + group: redis + mode: 0600 + +- name: Create Redis service file + copy: + src: redis.service + dest: /etc/systemd/system/redis.service + owner: root + group: root + mode: 0644 + +- name: Add service file for disabling transparent hugepage + copy: + src: transparent_hugepage.service + dest: /etc/systemd/system/transparent_hugepage.service + owner: root + group: root + mode: 0644 + +- name: Check if transparent hugepage is currently enabled + command: + cmd: cat /sys/kernel/mm/transparent_hugepage/enabled + register: transparent_hugepage + changed_when: false + +- name: Start and enable "disable transparent hugepage" service + service: + name: transparent_hugepage.service + state: started + enabled: true + when: "'[never]' not in transparent_hugepage.stdout" + +- name: Check if kernel overcommit mode is already set + command: + cmd: sysctl -n vm.overcommit_memory + register: overcommit_memory + changed_when: false + +- name: Set kernel overcommit mode temporarily (recommended by Redis) + command: + cmd: sysctl vm.overcommit_memory=1 + when: overcommit_memory.stdout == "0" + +- name: Make kernel overcommit mode permanent (recommended by Redis, requires restart) + lineinfile: + path: /etc/sysctl.conf + line: vm.overcommit_memory = 1 + +- name: Start and enable redis service + service: + name: redis + state: started + enabled: true diff --git a/salt/salt/redis/redis.conf.jinja2 b/ansible/roles/redis/templates/redis.conf.jinja2 similarity index 99% rename from salt/salt/redis/redis.conf.jinja2 rename to ansible/roles/redis/templates/redis.conf.jinja2 index d032c42..0c597d3 100644 --- a/salt/salt/redis/redis.conf.jinja2 +++ b/ansible/roles/redis/templates/redis.conf.jinja2 @@ -43,12 +43,6 @@ # loadmodule /path/to/my_module.so # loadmodule /path/to/other_module.so -{% if accumulator is defined %} -{% for line in accumulator['redis_loadmodule_lines'] -%} -{{ line }} -{% endfor %} -{% endif %} - ################################## NETWORK ##################################### # By default, if no "bind" configuration directive is specified, Redis listens diff --git a/ansible/roles/redis_module_cell/meta/main.yml b/ansible/roles/redis_module_cell/meta/main.yml new file mode 100644 index 0000000..89a7c92 --- /dev/null +++ b/ansible/roles/redis_module_cell/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: redis diff --git a/ansible/roles/redis_module_cell/tasks/main.yml b/ansible/roles/redis_module_cell/tasks/main.yml new file mode 100644 index 0000000..9ef2d62 --- /dev/null +++ b/ansible/roles/redis_module_cell/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Download redis-cell Redis module from GitHub + get_url: + dest: /tmp/redis-cell.tar.gz + url: https://github.com/brandur/redis-cell/releases/download/v0.2.1/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz + checksum: sha256:9427fb100f4cada817f30f854ead7f233de32948a0ec644f15988c275a2ed1cb + +- name: Create /opt/redis-cell + file: + path: /opt/redis-cell + state: directory + owner: redis + group: redis + mode: 0755 + +- name: Extract redis-cell + unarchive: + remote_src: true + src: /tmp/redis-cell.tar.gz + dest: /opt/redis-cell + +- name: Load redis-cell module in Redis configuration + lineinfile: + path: /etc/redis.conf + line: loadmodule /opt/redis-cell/libredis_cell.so + notify: + - Restart redis diff --git a/ansible/roles/scripts/tasks/main.yml b/ansible/roles/scripts/tasks/main.yml new file mode 100644 index 0000000..cb9890d --- /dev/null +++ b/ansible/roles/scripts/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Install the activate script + template: + src: activate.sh.jinja2 + dest: /usr/local/bin/activate + owner: root + group: root + mode: 0755 diff --git a/salt/salt/scripts/activate.sh.jinja2 b/ansible/roles/scripts/templates/activate.sh.jinja2 similarity index 78% rename from salt/salt/scripts/activate.sh.jinja2 rename to ansible/roles/scripts/templates/activate.sh.jinja2 index 014536c..21871df 100644 --- a/salt/salt/scripts/activate.sh.jinja2 +++ b/ansible/roles/scripts/templates/activate.sh.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} #!/bin/bash # # Simple convenience script to activate the venv and switch to the app dir diff --git a/ansible/roles/self_signed_ssl_cert/meta/main.yml b/ansible/roles/self_signed_ssl_cert/meta/main.yml new file mode 100644 index 0000000..8b662c9 --- /dev/null +++ b/ansible/roles/self_signed_ssl_cert/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: nginx diff --git a/ansible/roles/self_signed_ssl_cert/tasks/main.yml b/ansible/roles/self_signed_ssl_cert/tasks/main.yml new file mode 100644 index 0000000..e9bd31c --- /dev/null +++ b/ansible/roles/self_signed_ssl_cert/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Install packages needed by Ansible community plugins + pip: + executable: pip3 + name: cryptography + +- name: Create directory for certificate + file: + path: "{{ ssl_cert_dir }}" + state: directory + mode: 0755 + +- name: Create a private key + community.crypto.openssl_privatekey: + path: "{{ ssl_private_key_path }}" + +- name: Create a self-signed certificate + community.crypto.x509_certificate: + path: "{{ ssl_cert_path }}" + privatekey_path: "{{ ssl_private_key_path }}" + provider: selfsigned + notify: + - Reload nginx diff --git a/ansible/roles/webassets/meta/main.yml b/ansible/roles/webassets/meta/main.yml new file mode 100644 index 0000000..f7230ad --- /dev/null +++ b/ansible/roles/webassets/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: python diff --git a/ansible/roles/webassets/tasks/main.yml b/ansible/roles/webassets/tasks/main.yml new file mode 100644 index 0000000..0ce8619 --- /dev/null +++ b/ansible/roles/webassets/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Check if site-icons.css file exists + stat: + path: "{{ app_dir }}/static/css/site-icons.css" + register: site_icons_css_file + +# webassets will crash the site if this file doesn't exist +- name: Create site-icons.css file + file: + path: "{{ app_dir }}/static/css/site-icons.css" + state: touch + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0644 + when: not site_icons_css_file.stat.exists + +- name: Create systemd service file + template: + src: webassets.service.jinja2 + dest: /etc/systemd/system/webassets.service + owner: root + group: root + mode: 0644 + +- name: Start and enable webassets service + service: + name: webassets + state: started + enabled: true diff --git a/salt/salt/webassets.service.jinja2 b/ansible/roles/webassets/templates/webassets.service.jinja2 similarity index 80% rename from salt/salt/webassets.service.jinja2 rename to ansible/roles/webassets/templates/webassets.service.jinja2 index 1c3ad7d..f84355a 100644 --- a/salt/salt/webassets.service.jinja2 +++ b/ansible/roles/webassets/templates/webassets.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Webassets - auto-compile JS files on change diff --git a/ansible/roles/wiki_repo/tasks/main.yml b/ansible/roles/wiki_repo/tasks/main.yml new file mode 100644 index 0000000..e5ed427 --- /dev/null +++ b/ansible/roles/wiki_repo/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: Create the base repo directory + file: + path: /var/lib/tildes-wiki + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0775 + +- name: Check if a wiki git repo exists + stat: + path: /var/lib/tildes-wiki/.git + register: wiki_repo + +- name: Create a git repo and initial commit + become_user: "{{ app_username }}" + shell: + cmd: | + git init + git config user.name "Tildes" + git config user.email "Tildes" + git commit --allow-empty -m "Initial commit" + chdir: /var/lib/tildes-wiki + when: not wiki_repo.stat.exists diff --git a/ansible/tasks/prometheus_user.yml b/ansible/tasks/prometheus_user.yml new file mode 100644 index 0000000..36a3c19 --- /dev/null +++ b/ansible/tasks/prometheus_user.yml @@ -0,0 +1,10 @@ +--- +- name: Create group for prometheus user + group: + name: prometheus + +- name: Create prometheus user + user: + name: prometheus + group: prometheus + create_home: false diff --git a/ansible/vars.yml b/ansible/vars.yml new file mode 100644 index 0000000..4012c89 --- /dev/null +++ b/ansible/vars.yml @@ -0,0 +1,8 @@ +--- +app_dir: /opt/tildes +venv_dir: /opt/venvs/tildes +bin_dir: "{{ venv_dir }}/bin" + +static_sites_dir: /opt/tildes-static-sites + +python_version: 3.8 diff --git a/salt/minion b/salt/minion deleted file mode 100644 index fe4e094..0000000 --- a/salt/minion +++ /dev/null @@ -1,19 +0,0 @@ -# look for files on the minion, not the master -file_client: local - -# set a specific minion ID for use in top file / pillars -# options: -# - "dev" for the vagrant setup -# - "prod" for (single-server) production -# - "monitoring" for the monitoring server -id: dev - -# base path for SSL certificates -ca.cert_base_path: '/etc/pki' - -state_verbose: False - -# enable new module.run state syntax -# https://docs.saltstack.com/en/develop/topics/releases/2017.7.0.html#state-module-changes -use_superseded: - - module.run diff --git a/salt/pillar/dev.sls b/salt/pillar/dev.sls deleted file mode 100644 index 6447f76..0000000 --- a/salt/pillar/dev.sls +++ /dev/null @@ -1,8 +0,0 @@ -gunicorn_args: --reload -ini_file: development.ini -ssl_cert_path: /etc/pki/tls/certs/localhost.crt -ssl_private_key_path: /etc/pki/tls/certs/localhost.key -nginx_worker_processes: 1 -postgresql_version: 12 -prometheus_ips: ['127.0.0.1'] -site_hostname: localhost diff --git a/salt/pillar/monitoring-secrets.example.sls b/salt/pillar/monitoring-secrets.example.sls deleted file mode 100644 index c505701..0000000 --- a/salt/pillar/monitoring-secrets.example.sls +++ /dev/null @@ -1,8 +0,0 @@ -sentry_secret: 'sentry_secret_token' -sentry_email: 'sentry_superuser@email.address' -sentry_password: 'passwordforsentrysuperuser' -sentry_server_name: 'sentry.example.com' -developer_ips: ['127.0.0.1'] -prometheus_server_name: 'prom.example.com' -grafana_server_name: 'grafana.example.com' -grafana_admin_password: 'passwordforgrafanaadmin' diff --git a/salt/pillar/monitoring.sls b/salt/pillar/monitoring.sls deleted file mode 100644 index 004b89a..0000000 --- a/salt/pillar/monitoring.sls +++ /dev/null @@ -1,6 +0,0 @@ -ssl_cert_path: /etc/letsencrypt/live/tildes.net/fullchain.pem -ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem -hsts_max_age: 60 -nginx_worker_processes: auto -postgresql_version: 9.6 -site_hostname: tildes.net diff --git a/salt/pillar/prod.sls b/salt/pillar/prod.sls deleted file mode 100644 index a03a6d6..0000000 --- a/salt/pillar/prod.sls +++ /dev/null @@ -1,11 +0,0 @@ -gunicorn_args: --workers 8 -ini_file: production.ini -ssl_cert_path: /etc/letsencrypt/live/tildes.net/fullchain.pem -ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem -hsts_max_age: 63072000 -nginx_worker_processes: auto -postgresql_version: 12 -prometheus_ips: ['2607:5300:201:3100::6e77'] -site_hostname: tildes.net -ipv6_address: '2607:5300:0203:2dd8::' -ipv6_gateway: '2607:5300:0203:2dff:ff:ff:ff:ff' diff --git a/salt/pillar/top.sls b/salt/pillar/top.sls deleted file mode 100644 index 91f58d9..0000000 --- a/salt/pillar/top.sls +++ /dev/null @@ -1,8 +0,0 @@ -base: - 'dev': - - dev - 'prod': - - prod - 'monitoring': - - monitoring - - monitoring-secrets diff --git a/salt/salt/boussole.sls b/salt/salt/boussole.sls deleted file mode 100644 index 9597569..0000000 --- a/salt/salt/boussole.sls +++ /dev/null @@ -1,47 +0,0 @@ -{% from 'common.jinja2' import app_dir, python_version %} - -{% set boussole_venv_dir = '/opt/venvs/boussole' %} - -# Salt seems to use the deprecated pyvenv script, manual for now -boussole-venv-setup: - cmd.run: - - name: python{{ python_version }} -m venv {{ boussole_venv_dir }} - - creates: {{ boussole_venv_dir }} - - require: - - pkg: python{{ python_version }}-venv - -boussole-pip-installs: - cmd.run: - - name: {{ boussole_venv_dir }}/bin/pip install boussole - - unless: ls {{ boussole_venv_dir }}/lib/python{{ python_version }}/site-packages/boussole - -/etc/systemd/system/boussole.service: - file.managed: - - source: salt://boussole.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: boussole.service - -boussole.service: - service.running: - - enable: True - - require: - - cmd: boussole-pip-installs - -create-css-directory: - file.directory: - - name: {{ app_dir }}/static/css - -initial-boussole-run: - cmd.run: - - name: {{ boussole_venv_dir }}/bin/boussole compile --backend=yaml --config=boussole.yaml - - cwd: {{ app_dir }} - - env: - - LC_ALL: C.UTF-8 - - LANG: C.UTF-8 - - require: - - file: create-css-directory - - unless: ls {{ app_dir }}/static/css/*.css diff --git a/salt/salt/cmark-gfm.sls b/salt/salt/cmark-gfm.sls deleted file mode 100644 index e0928b5..0000000 --- a/salt/salt/cmark-gfm.sls +++ /dev/null @@ -1,25 +0,0 @@ -unpack-cmark-gfm: - archive.extracted: - - name: /tmp/cmark-gfm - - source: - - salt://cmark-gfm.tar.gz - - https://github.com/github/cmark-gfm/archive/0.29.0.gfm.0.tar.gz - - source_hash: sha256=6a94aeaa59a583fadcbf28de81dea8641b3f56d935dda5b2447a3c8df6c95fea - - if_missing: /usr/local/lib/libcmark-gfm.so - - options: --strip-components=1 - - enforce_toplevel: False - -install-cmark-build-deps: - pkg.installed: - - name: cmake - -install-cmark-gfm: - cmd.run: - - cwd: /tmp/cmark-gfm/ - - names: - - make - - make install - - onchanges: - - archive: unpack-cmark-gfm - - require: - - pkg: install-cmark-build-deps diff --git a/salt/salt/common.jinja2 b/salt/salt/common.jinja2 deleted file mode 100644 index ee73a10..0000000 --- a/salt/salt/common.jinja2 +++ /dev/null @@ -1,13 +0,0 @@ -{% set python_version = '3.8' %} - -{% set app_dir = '/opt/tildes' %} -{% set static_sites_dir = '/opt/tildes-static-sites' %} - -{% set venv_dir = '/opt/venvs/tildes' %} -{% set bin_dir = venv_dir + '/bin' %} - -{% if grains['id'] == 'dev' %} - {% set app_username = 'vagrant' %} -{% else %} - {% set app_username = 'tildes' %} -{% endif %} diff --git a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 b/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 deleted file mode 100644 index 352eb76..0000000 --- a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Comment User Mention Generator (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python comment_user_mentions_generator.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/init.sls b/salt/salt/consumers/init.sls deleted file mode 100644 index 5c4df93..0000000 --- a/salt/salt/consumers/init.sls +++ /dev/null @@ -1,85 +0,0 @@ -/etc/systemd/system/consumer-topic_interesting_activity_updater.service: - file.managed: - - source: salt://consumers/topic_interesting_activity_updater.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-topic_metadata_generator.service: - file.managed: - - source: salt://consumers/topic_metadata_generator.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-comment_user_mentions_generator.service: - file.managed: - - source: salt://consumers/comment_user_mentions_generator.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-post_processing_script_runner.service: - file.managed: - - source: salt://consumers/post_processing_script_runner.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_interesting_activity_updater.service: - service.running: - - enable: True - -consumer-topic_metadata_generator.service: - service.running: - - enable: True - -consumer-comment_user_mentions_generator.service: - service.running: - - enable: True - -consumer-post_processing_script_runner.service: - service.running: - - enable: True - -{% if grains['id'] == 'prod' %} -/etc/systemd/system/consumer-topic_embedly_extractor.service: - file.managed: - - source: salt://consumers/topic_embedly_extractor.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_embedly_extractor.service: - service.running: - - enable: True - -/etc/systemd/system/consumer-topic_youtube_scraper.service: - file.managed: - - source: salt://consumers/topic_youtube_scraper.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_youtube_scraper.service: - service.running: - - enable: True - -/etc/systemd/system/consumer-site_icon_downloader.service: - file.managed: - - source: salt://consumers/site_icon_downloader.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-site_icon_downloader.service: - service.running: - - enable: True -{% endif %} diff --git a/salt/salt/consumers/post_processing_script_runner.service.jinja2 b/salt/salt/consumers/post_processing_script_runner.service.jinja2 deleted file mode 100644 index b7c0f34..0000000 --- a/salt/salt/consumers/post_processing_script_runner.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Post Processing Script Runner (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python post_processing_script_runner.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/site_icon_downloader.service.jinja2 b/salt/salt/consumers/site_icon_downloader.service.jinja2 deleted file mode 100644 index ee7a98a..0000000 --- a/salt/salt/consumers/site_icon_downloader.service.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Site Icon Downloader (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python site_icon_downloader.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 b/salt/salt/consumers/topic_embedly_extractor.service.jinja2 deleted file mode 100644 index a0061ea..0000000 --- a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Embedly Extractor (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_embedly_extractor.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 b/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 deleted file mode 100644 index 0cf8fc5..0000000 --- a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Interesting Activity Updater (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_interesting_activity_updater.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_metadata_generator.service.jinja2 b/salt/salt/consumers/topic_metadata_generator.service.jinja2 deleted file mode 100644 index 0d20257..0000000 --- a/salt/salt/consumers/topic_metadata_generator.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Metadata Generator (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_metadata_generator.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/cronjobs.sls b/salt/salt/cronjobs.sls deleted file mode 100644 index 2d561ad..0000000 --- a/salt/salt/cronjobs.sls +++ /dev/null @@ -1,46 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir %} - -lift-expired-temp-bans-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.lift_expired_temporary_bans import lift_expired_temporary_bans; lift_expired_temporary_bans('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 1 - -close-voting-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.close_voting_on_old_posts import close_voting_on_old_posts; close_voting_on_old_posts('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 3 - -data-cleanup-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.clean_private_data import clean_all_data; clean_all_data('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - hour: 4 - - minute: 10 - -generate-group-stats-for-yesterday-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.generate_group_stats_for_yesterday import generate_stats; generate_stats('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: 0 - - minute: 10 - -generate-site-icons-css-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.generate_site_icons_css import generate_css; generate_css()" - - user: {{ app_username }} - - minute: '*/5' - -post-scheduled-topics-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - -update-common-topic-tags-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 0 diff --git a/salt/salt/development.sls b/salt/salt/development.sls deleted file mode 100644 index bc4776e..0000000 --- a/salt/salt/development.sls +++ /dev/null @@ -1,27 +0,0 @@ -{% from 'common.jinja2' import app_username, bin_dir %} - -{% set profile_dir = '/home/' + app_username + '/.ipython/profile_default' %} - -ipython-profile: - cmd.run: - - name: {{ bin_dir }}/ipython profile create - - runas: {{ app_username }} - - creates: {{ profile_dir }} - file.managed: - - name: {{ profile_dir }}/ipython_config.py - - contents: - - c.InteractiveShellApp.extensions = ['autoreload'] - - c.InteractiveShellApp.exec_lines = ['%autoreload 2'] - require: - - pip: pip-installs - -automatic-activate: - file.append: - - name: '/home/{{ app_username }}/.bashrc' - - text: 'source activate' - -# adds invoke's tab-completion script so that invoke tasks can be completed -invoke-tab-completion: - file.append: - - name: '/home/{{ app_username }}/.bashrc' - - text: 'source <(invoke --print-completion-script bash)' diff --git a/salt/salt/final-setup.sls b/salt/salt/final-setup.sls deleted file mode 100644 index 6f3cb48..0000000 --- a/salt/salt/final-setup.sls +++ /dev/null @@ -1,19 +0,0 @@ -{% from 'common.jinja2' import app_dir, bin_dir %} - -initialize-db: - cmd.run: - - name: {{ bin_dir }}/python -c "from scripts.initialize_db import initialize_db; initialize_db('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - cwd: {{ app_dir }} - - runas: postgres - - require: - - postgres_database: tildes - - unless: psql -U tildes tildes -c "SELECT user_id FROM users;" - -{% if grains['id'] == 'dev' %} -insert-dev-data: - cmd.run: - - name: {{ bin_dir }}/python -c "from scripts.initialize_db import insert_dev_data; insert_dev_data('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - cwd: {{ app_dir }} - - onchanges: - - cmd: initialize-db -{% endif %} diff --git a/salt/salt/general-config.sls b/salt/salt/general-config.sls deleted file mode 100644 index 22c8f68..0000000 --- a/salt/salt/general-config.sls +++ /dev/null @@ -1,3 +0,0 @@ -set-timezone-to-utc: - timezone.system: - - name: 'Etc/UTC' diff --git a/salt/salt/grafana/grafana.conf.jinja2 b/salt/salt/grafana/grafana.conf.jinja2 deleted file mode 100644 index 480d59a..0000000 --- a/salt/salt/grafana/grafana.conf.jinja2 +++ /dev/null @@ -1,19 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['grafana_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_redirect off; - proxy_pass http://localhost:3000; - } -} diff --git a/salt/salt/grafana/grafana.ini.jinja2 b/salt/salt/grafana/grafana.ini.jinja2 deleted file mode 100644 index db942d8..0000000 --- a/salt/salt/grafana/grafana.ini.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -[server] -domain = {{ pillar['grafana_server_name'] }} -root_url = https://{{ pillar['grafana_server_name'] }} - -[analytics] -reporting_enabled = false - -[security] -admin_password = """{{ pillar['grafana_admin_password'] }}""" -disable_gravatar = true - -[snapshots] -external_enabled = false - -[users] -allow_sign_up = false -allow_org_create = false diff --git a/salt/salt/grafana/init.sls b/salt/salt/grafana/init.sls deleted file mode 100644 index 7c556bd..0000000 --- a/salt/salt/grafana/init.sls +++ /dev/null @@ -1,61 +0,0 @@ -grafana: - pkgrepo.managed: - - name: deb https://packages.grafana.com/oss/deb stable main - - dist: stable - - file: /etc/apt/sources.list.d/grafana.list - - key_url: https://packages.grafana.com/gpg.key - - require_in: - - pkg: grafana - pkg.installed: - - name: grafana - - refresh: True - # note: this file must be set up before the server is started for the first - # time, otherwise the admin password will not be set correctly - file.managed: - - name: /etc/grafana/grafana.ini - - source: salt://grafana/grafana.ini.jinja2 - - template: jinja - - user: root - - group: grafana - - mode: 640 - service.running: - - name: grafana-server - - enable: True - - require: - - pkg: grafana - - file: /etc/grafana/grafana.ini - - watch: - - file: /etc/grafana/grafana.ini - http.query: - - name: http://localhost:3000/api/datasources - - method: POST - - username: admin - - password: {{ pillar['grafana_admin_password'] }} - - data: | - {"name": "Prometheus", - "type": "prometheus", - "url": "http://localhost:9090", - "access": "proxy", - "isDefault": true} - - header_list: - - 'Content-Type: application/json' - - status: 200 - - unless: - - curl -f http://admin:{{ pillar['grafana_admin_password'] }}@localhost:3000/api/datasources/name/Prometheus - -/etc/nginx/sites-available/grafana.conf: - file.managed: - - source: salt://grafana/grafana.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/grafana.conf: - file.symlink: - - target: /etc/nginx/sites-available/grafana.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/gunicorn/init.sls b/salt/salt/gunicorn/init.sls deleted file mode 100644 index bcaa488..0000000 --- a/salt/salt/gunicorn/init.sls +++ /dev/null @@ -1,59 +0,0 @@ -{% from 'common.jinja2' import app_username %} - -/etc/systemd/system/gunicorn.service: - file.managed: - - source: salt://gunicorn/gunicorn.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -/etc/systemd/system/gunicorn.socket: - file.managed: - - source: salt://gunicorn/gunicorn.socket.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -/usr/lib/tmpfiles.d/gunicorn.conf: - file.managed: - - source: salt://gunicorn/gunicorn.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -gunicorn.socket: - service.running: - - enable: True - -# Set up the gunicorn_reloader service, which reloads gunicorn whenever certain files -# are changed (such as static files, to update the cache-busting strings) -/etc/systemd/system/gunicorn_reloader.service: - file.managed: - - source: salt://gunicorn/gunicorn_reloader.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn_reloader.path - -/etc/systemd/system/gunicorn_reloader.path: - file.managed: - - source: salt://gunicorn/gunicorn_reloader.path - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn_reloader.path - -gunicorn_reloader.path: - service.running: - - enable: True diff --git a/salt/salt/java.sls b/salt/salt/java.sls deleted file mode 100644 index 2faa64f..0000000 --- a/salt/salt/java.sls +++ /dev/null @@ -1,3 +0,0 @@ -java-openjdk: - pkg.installed: - - name: openjdk-8-jre diff --git a/salt/salt/nginx/init.sls b/salt/salt/nginx/init.sls deleted file mode 100644 index 1b84a6d..0000000 --- a/salt/salt/nginx/init.sls +++ /dev/null @@ -1,33 +0,0 @@ -nginx: - pkgrepo.managed: - - name: deb http://nginx.org/packages/ubuntu/ xenial nginx - - dist: xenial - - file: /etc/apt/sources.list.d/nginx.list - - key_url: https://nginx.org/keys/nginx_signing.key - - require_in: - - pkg: nginx - pkg.installed: - - name: nginx - - refresh: True - service.running: - - require: - - pkg: nginx - - reload: True - - watch: - - file: /etc/nginx/* - -/etc/nginx/nginx.conf: - file.managed: - - source: salt://nginx/nginx.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -# Add logrotate config to rotate daily and delete after 30 days -/etc/logrotate.d/nginx: - file.managed: - - source: salt://nginx/logrotate - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/shortener-config.sls b/salt/salt/nginx/shortener-config.sls deleted file mode 100644 index 03cc034..0000000 --- a/salt/salt/nginx/shortener-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes-shortener.conf: - file.managed: - - source: salt://nginx/tildes-shortener.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes-shortener.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes-shortener.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/site-config.sls b/salt/salt/nginx/site-config.sls deleted file mode 100644 index e42e420..0000000 --- a/salt/salt/nginx/site-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes.conf: - file.managed: - - source: salt://nginx/tildes.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/static-sites-config.sls b/salt/salt/nginx/static-sites-config.sls deleted file mode 100644 index 2d034b6..0000000 --- a/salt/salt/nginx/static-sites-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes-static-sites.conf: - file.managed: - - source: salt://nginx/tildes-static-sites.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes-static-sites.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes-static-sites.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nodejs.sls b/salt/salt/nodejs.sls deleted file mode 100644 index b6ab35f..0000000 --- a/salt/salt/nodejs.sls +++ /dev/null @@ -1,25 +0,0 @@ -{% from "common.jinja2" import app_dir, app_username %} - -# Add the NodeSource repository and install Node.js 10.x -nodejs-pkgrepo: - pkgrepo.managed: - - name: deb https://deb.nodesource.com/node_10.x xenial main - - dist: xenial - - file: /etc/apt/sources.list.d/nodesource.list - - key_url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key - - require_in: - - pkg: nodejs - pkg.installed: - - name: nodejs - - refresh: True - -# Install the npm packages defined in package.json -# Uses the --no-bin-links option to prevent npm from creating symlinks in the .bin -# directory, which doesn't work inside Vagrant on Windows -install-npm-packages: - cmd.run: - - name: npm install --no-bin-links - - cwd: {{ app_dir }} - - runas: {{ app_username }} - - require: - - pkg: nodejs diff --git a/salt/salt/postgresql-redis-bridge.sls b/salt/salt/postgresql-redis-bridge.sls deleted file mode 100644 index 411e6cd..0000000 --- a/salt/salt/postgresql-redis-bridge.sls +++ /dev/null @@ -1,14 +0,0 @@ -/etc/systemd/system/postgresql_redis_bridge.service: - file.managed: - - source: salt://postgresql_redis_bridge.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require: - - service: redis.service - - service: postgresql - -postgresql_redis_bridge.service: - service.running: - - enable: True diff --git a/salt/salt/postgresql/init.sls b/salt/salt/postgresql/init.sls deleted file mode 100644 index b251d81..0000000 --- a/salt/salt/postgresql/init.sls +++ /dev/null @@ -1,85 +0,0 @@ -{% from 'common.jinja2' import app_dir, python_version, venv_dir %} - -postgresql: - pkgrepo.managed: - - name: deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main - - dist: xenial-pgdg - - file: /etc/apt/sources.list.d/psql.list - - key_url: https://www.postgresql.org/media/keys/ACCC4CF8.asc - - require_in: - - pkg: postgresql - pkg.installed: - - name: postgresql-{{ pillar['postgresql_version'] }} - - refresh: True - service.running: - - require: - - pkg: postgresql - - reload: True - - watch: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/*.conf - -install-plpython3: - pkg.installed: - - name: postgresql-plpython3-{{ pillar['postgresql_version'] }} - - refresh: True - - require: - - pkgrepo: postgresql - -set-lock-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?lock_timeout = (?!5000).*$' - - repl: 'lock_timeout = 5000' - -set-statement-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?statement_timeout = (?!5000).*$' - - repl: 'statement_timeout = 5000' - -set-idle-in-transaction-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?idle_in_transaction_session_timeout = (?!600000).*$' - - repl: 'idle_in_transaction_session_timeout = 600000' - -set-timezone: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?timezone = (?!''UTC'').*$' - - repl: 'timezone = ''UTC''' - -enable-pg-stat-statements: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?shared_preload_libraries = .*$' - - repl: "shared_preload_libraries = 'pg_stat_statements'" - - watch_in: - - module: enable-pg-stat-statements - module.wait: - - service.restart: - - name: postgresql.service - -/etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf: - file.managed: - - source: salt://postgresql/pg_hba.conf.jinja2 - - template: jinja - - user: postgres - - group: postgres - - mode: 640 - require: - - service: postgresql - -# set PYTHONPATH env var in postgresql so PL/Python can access all the modules -set-postgresql-pythonpath: - file.managed: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/environment - - contents: "PYTHONPATH='{{ venv_dir }}/lib/python{{ python_version }}/site-packages:{{ app_dir }}'" - - user: postgres - - group: postgres - - mode: 644 - - watch_in: - - module: set-postgresql-pythonpath - module.wait: - - service.restart: - - name: postgresql.service diff --git a/salt/salt/postgresql/pg_hba.conf.jinja2 b/salt/salt/postgresql/pg_hba.conf.jinja2 deleted file mode 100644 index 45e7ba9..0000000 --- a/salt/salt/postgresql/pg_hba.conf.jinja2 +++ /dev/null @@ -1,6 +0,0 @@ -{% for line in accumulator['pg_hba_lines'] -%} -{{ line }} -{% endfor %} - -# Required: allow the database superuser to connect via socket -local all postgres peer diff --git a/salt/salt/postgresql/pgbouncer.sls b/salt/salt/postgresql/pgbouncer.sls deleted file mode 100644 index 21d7453..0000000 --- a/salt/salt/postgresql/pgbouncer.sls +++ /dev/null @@ -1,25 +0,0 @@ -install-pgbouncer: - pkg.installed: - - name: pgbouncer - -/etc/pgbouncer/pgbouncer.ini: - file.managed: - - source: salt://postgresql/pgbouncer.ini.jinja2 - - template: jinja - - user: postgres - - group: postgres - - mode: 640 - -/etc/pgbouncer/userlist.txt: - file.managed: - - contents: '"tildes" ""' - - user: postgres - - group: postgres - - mode: 640 - -pgbouncer.service: - service.running: - - enable: True - - reload: True - - watch: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/postgresql/prod-config.sls b/salt/salt/postgresql/prod-config.sls deleted file mode 100644 index 40c477d..0000000 --- a/salt/salt/postgresql/prod-config.sls +++ /dev/null @@ -1,25 +0,0 @@ -# Values from PGTune (https://pgtune.leopard.in.ua/) -{% set setting_values = { - "checkpoint_completion_target": "0.7", - "default_statistics_target": "100", - "effective_cache_size": "24GB", - "effective_io_concurrency": 200, - "maintenance_work_mem": "2GB", - "max_parallel_workers": "8", - "max_parallel_workers_per_gather": "4", - "max_wal_size": "2GB", - "max_worker_processes": "8", - "min_wal_size": "1GB", - "random_page_cost": "1.1", - "shared_buffers": "8GB", - "wal_buffers": "16MB", - "work_mem": "10485kB", -} %} - -{% for setting, value in setting_values.items() %} -postgresql-conf-set-{{ setting }}: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?{{ setting }} = (?!{{ value }}).*$' - - repl: '{{ setting }} = {{ value }}' -{% endfor %} diff --git a/salt/salt/postgresql/site-db.sls b/salt/salt/postgresql/site-db.sls deleted file mode 100644 index 52b0fef..0000000 --- a/salt/salt/postgresql/site-db.sls +++ /dev/null @@ -1,73 +0,0 @@ -site-db-user: - postgres_user.present: - - name: tildes - # set the tildes user to a database superuser in the dev environment only - - superuser: {{ grains['id'] == 'dev' }} - - require: - - service: postgresql - -site-db-database: - postgres_database.present: - - name: tildes - - owner: tildes - - require: - - postgres_user: tildes - -site-db-enable-citext: - postgres_extension.present: - - name: citext - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-ltree: - postgres_extension.present: - - name: ltree - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-intarray: - postgres_extension.present: - - name: intarray - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-pg_stat_statements: - postgres_extension.present: - - name: pg_stat_statements - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-pg_trgm: - postgres_extension.present: - - name: pg_trgm - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-plpython3u: - postgres_extension.present: - - name: plpython3u - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-pg_hba-lines: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: - - 'local sameuser tildes trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -site-db-pgbouncer-lines: - file.accumulated: - - name: pgbouncer_db_lines - - filename: /etc/pgbouncer/pgbouncer.ini - - text: 'tildes =' - - require_in: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/postgresql/test-db.sls b/salt/salt/postgresql/test-db.sls deleted file mode 100644 index 9f868d9..0000000 --- a/salt/salt/postgresql/test-db.sls +++ /dev/null @@ -1,58 +0,0 @@ -test-db-database: - postgres_database.present: - - name: tildes_test - - owner: tildes - - require: - - postgres_user: tildes - -test-db-enable-citext: - postgres_extension.present: - - name: citext - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-ltree: - postgres_extension.present: - - name: ltree - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-intarray: - postgres_extension.present: - - name: intarray - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-pg_trgm: - postgres_extension.present: - - name: pg_trgm - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-plpython3u: - postgres_extension.present: - - name: plpython3u - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-pg_hba-lines: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: - - 'local tildes_test tildes trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -test-db-pgbouncer-lines: - file.accumulated: - - name: pgbouncer_db_lines - - filename: /etc/pgbouncer/pgbouncer.ini - - text: 'tildes_test =' - - require_in: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/prod-config.sls b/salt/salt/prod-config.sls deleted file mode 100644 index 3e1ee10..0000000 --- a/salt/salt/prod-config.sls +++ /dev/null @@ -1,17 +0,0 @@ -# Force apt not to use IPv6 (seems to hang often) -/etc/apt/apt.conf.d/99force-ipv4: - file.managed: - - contents: 'Acquire::ForceIPv4 "true";' - -# Enable ipv6 networking -/etc/network/interfaces: - file.append: - - text: | - iface eth0 inet6 static - address {{ pillar['ipv6_address'] }} - netmask 64 - - post-up sleep 5; /sbin/ip -family inet6 route add {{ pillar['ipv6_gateway'] }} dev eth0 - post-up sleep 5; /sbin/ip -family inet6 route add default via {{ pillar['ipv6_gateway'] }} - pre-down /sbin/ip -family inet6 route del default via {{ pillar['ipv6_gateway'] }} - pre-down /sbin/ip -family inet6 route del {{ pillar['ipv6_gateway'] }} dev eth0 diff --git a/salt/salt/prometheus/exporters/blackbox.yml b/salt/salt/prometheus/exporters/blackbox.yml deleted file mode 100644 index 33cf2cb..0000000 --- a/salt/salt/prometheus/exporters/blackbox.yml +++ /dev/null @@ -1,14 +0,0 @@ -modules: - site_ipv4: - prober: http - timeout: 5s - http: - preferred_ip_protocol: "ip4" - ip_protocol_fallback: false - - site_ipv6: - prober: http - timeout: 5s - http: - preferred_ip_protocol: "ip6" - ip_protocol_fallback: false diff --git a/salt/salt/prometheus/exporters/blackbox_exporter.sls b/salt/salt/prometheus/exporters/blackbox_exporter.sls deleted file mode 100644 index 42884b8..0000000 --- a/salt/salt/prometheus/exporters/blackbox_exporter.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Download/extract and set up the blackbox exporter -include: - - prometheus.user - -unpack-blackbox-exporter: - archive.extracted: - - name: /opt/prometheus_blackbox_exporter - - source: - - salt://prometheus/exporters/blackbox_exporter-0.16.0.linux-amd64.tar.gz - - https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz - - source_hash: sha256=52d3444a518ea01f220e08eaa53eb717ef54da6724760c925ab41285d0d5a7bd - - if_missing: /opt/prometheus_blackbox_exporter - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/opt/prometheus_blackbox_exporter/blackbox.yml: - file.managed: - - source: salt://prometheus/exporters/blackbox.yml - - user: prometheus - - group: prometheus - - mode: 644 - -/etc/systemd/system/prometheus_blackbox_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_blackbox_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-blackbox-exporter-service: - service.running: - - name: prometheus_blackbox_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/node_exporter.sls b/salt/salt/prometheus/exporters/node_exporter.sls deleted file mode 100644 index 06d41b8..0000000 --- a/salt/salt/prometheus/exporters/node_exporter.sls +++ /dev/null @@ -1,28 +0,0 @@ -# Download/extract and set up the node exporter (hardware/OS metrics) -include: - - prometheus.user - -unpack-node-exporter: - archive.extracted: - - name: /opt/prometheus_node_exporter - - source: - - salt://prometheus/exporters/node_exporter-0.13.0.linux-amd64.tar.gz - - https://github.com/prometheus/node_exporter/releases/download/v0.13.0/node_exporter-0.13.0.linux-amd64.tar.gz - - source_hash: sha256=2de5d1e51330c41588ed4c88bc531a3d2dccf6b4d7b99d5782d95cff27a3c049 - - if_missing: /opt/prometheus_node_exporter - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/etc/systemd/system/prometheus_node_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_node_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-node-exporter-service: - service.running: - - name: prometheus_node_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/postgres_exporter.sls b/salt/salt/prometheus/exporters/postgres_exporter.sls deleted file mode 100644 index 9c22869..0000000 --- a/salt/salt/prometheus/exporters/postgres_exporter.sls +++ /dev/null @@ -1,36 +0,0 @@ -# Download and set up the postgres exporter -include: - - prometheus.user - -unpack-postgres-exporter: - archive.extracted: - - name: /opt/prometheus_postgres_exporter - - source: - - salt://prometheus/exporters/postgres_exporter_v0.4.7_linux-amd64.tar.gz - - https://github.com/wrouesnel/postgres_exporter/releases/download/v0.4.7/postgres_exporter_v0.4.7_linux-amd64.tar.gz - - source_hash: sha256=c34d61bb4deba8efae06fd3c9979b96dae3f3c757698ce3384c80fff586c667b - - if_missing: /opt/prometheus_postgres_exporter - - user: postgres - - group: postgres - - mode: 774 - - options: --strip-components=1 - - enforce_toplevel: False - -/opt/prometheus_postgres_exporter/queries.yaml: - file.managed: - - source: salt://prometheus/exporters/queries.yaml - - user: postgres - - group: postgres - - mode: 644 - -/etc/systemd/system/prometheus_postgres_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_postgres_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-postgres-exporter-service: - service.running: - - name: prometheus_postgres_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service b/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service deleted file mode 100644 index 430c308..0000000 --- a/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Prometheus Blackbox Exporter -After=syslog.target network.target - -[Service] -Type=simple -RemainAfterExit=no -WorkingDirectory=/opt/prometheus_blackbox_exporter -User=prometheus -Group=prometheus -ExecStart=/opt/prometheus_blackbox_exporter/blackbox_exporter --config.file=/opt/prometheus_blackbox_exporter/blackbox.yml - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/prometheus/exporters/redis_exporter.sls b/salt/salt/prometheus/exporters/redis_exporter.sls deleted file mode 100644 index a731283..0000000 --- a/salt/salt/prometheus/exporters/redis_exporter.sls +++ /dev/null @@ -1,27 +0,0 @@ -# Download/extract and set up the redis exporter -include: - - prometheus.user - -unpack-redis-exporter: - archive.extracted: - - name: /opt/prometheus_redis_exporter - - source: - - salt://prometheus/exporters/redis_exporter-v0.26.0.linux-amd64.tar.gz - - https://github.com/oliver006/redis_exporter/releases/download/v0.26.0/redis_exporter-v0.26.0.linux-amd64.tar.gz - - source_hash: sha256=39354c57b9d02b455c584baf46a2df6ed3d1ac190c88e3ec0fa0c23b49ccc2bb - - if_missing: /opt/prometheus_redis_exporter - - user: prometheus - - group: prometheus - - enforce_toplevel: False - -/etc/systemd/system/prometheus_redis_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_redis_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-redis-exporter-service: - service.running: - - name: prometheus_redis_exporter - - enable: True diff --git a/salt/salt/prometheus/init.sls b/salt/salt/prometheus/init.sls deleted file mode 100644 index 5001fe2..0000000 --- a/salt/salt/prometheus/init.sls +++ /dev/null @@ -1,58 +0,0 @@ -include: - - prometheus.user - -unpack-prometheus: - archive.extracted: - - name: /opt/prometheus - - source: - - salt://prometheus/prometheus-2.0.0.linux-amd64.tar.gz - - https://github.com/prometheus/prometheus/releases/download/v2.0.0/prometheus-2.0.0.linux-amd64.tar.gz - - source_hash: sha256=e12917b25b32980daee0e9cf879d9ec197e2893924bd1574604eb0f550034d46 - - if_missing: /opt/prometheus - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/etc/systemd/system/prometheus.service: - file.managed: - - source: salt://prometheus/prometheus.service - - user: root - - group: root - - mode: 644 - -prometheus-service: - service.running: - - name: prometheus - - enable: True - - watch: - - file: /opt/prometheus/prometheus.yml - - -/opt/prometheus/prometheus.yml: - file.managed: - - source: salt://prometheus/prometheus.yml.jinja2 - - template: jinja - - user: prometheus - - group: prometheus - - mode: 664 - -{# Set up nginx to reverse-proxy to prometheus in production #} -{% if grains['id'] == 'prod' %} -/etc/nginx/sites-available/prometheus.conf: - file.managed: - - source: salt://prometheus/prometheus.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/prometheus.conf: - file.symlink: - - target: /etc/nginx/sites-available/prometheus.conf - - makedirs: True - - user: root - - group: root - - mode: 644 -{% endif %} diff --git a/salt/salt/prometheus/prometheus.conf.jinja2 b/salt/salt/prometheus/prometheus.conf.jinja2 deleted file mode 100644 index 3b4134f..0000000 --- a/salt/salt/prometheus/prometheus.conf.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['prometheus_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - proxy_pass http://localhost:9090; - } -} diff --git a/salt/salt/prometheus/user.sls b/salt/salt/prometheus/user.sls deleted file mode 100644 index 46dcf72..0000000 --- a/salt/salt/prometheus/user.sls +++ /dev/null @@ -1,7 +0,0 @@ -prometheus-user: - group.present: - - name: prometheus - user.present: - - name: prometheus - - groups: [prometheus] - - createhome: False diff --git a/salt/salt/pts-lbsearch.sls b/salt/salt/pts-lbsearch.sls deleted file mode 100644 index d09686a..0000000 --- a/salt/salt/pts-lbsearch.sls +++ /dev/null @@ -1,12 +0,0 @@ -compile-pts-lbsearch: - file.managed: - - name: /tmp/pts_lbsearch.c - - source: - - https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c - - source_hash: sha256=ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1 - - creates: /usr/local/bin/pts_lbsearch - cmd.run: - - cwd: /tmp/ - # compilation command taken from the top of the source file - - name: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c - - creates: /usr/local/bin/pts_lbsearch diff --git a/salt/salt/python.sls b/salt/salt/python.sls deleted file mode 100644 index 5626bf8..0000000 --- a/salt/salt/python.sls +++ /dev/null @@ -1,60 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir, python_version, venv_dir %} - -deadsnakes: - pkgrepo.managed: - - ppa: deadsnakes/ppa - pkg.installed: - - name: python{{ python_version }} - - refresh: True - -/opt/venvs: - file.directory: - - user: {{ app_username }} - - group: {{ app_username }} - - dir_mode: 755 - -delete-obsolete-venv: - file.absent: - - name: {{ venv_dir }} - - unless: {{ bin_dir }}/python --version | grep {{ python_version }} - -# Salt seems to use the deprecated pyvenv script, manual for now -venv-setup: - pkg.installed: - - name: python{{ python_version }}-venv - cmd.run: - - name: python{{ python_version }} -m venv {{ venv_dir }} - - creates: {{ venv_dir }} - - runas: {{ app_username }} - - require: - - pkg: python{{ python_version }}-venv - -# Packages needed to be able to compile psycopg2 (while installing requirements.txt) -psycopg2-deps: - pkg.installed: - - pkgs: - - gcc - - libpq-dev - - python{{ python_version }}-dev - -pip-installs: - pip.installed: - {% if grains['id'] == 'dev' %} - - requirements: {{ app_dir }}/requirements-dev.txt - {% else %} - - requirements: {{ app_dir }}/requirements.txt - {% endif %} - - user: {{ app_username }} - - bin_env: {{ venv_dir }} - require: - - cmd: venv-setup - - pkg: psycopg2-deps - -self-install: - pip.installed: - - bin_env: {{ venv_dir }} - - editable: - - {{ app_dir }} - - require: - - cmd: venv-setup - - unless: ls {{ venv_dir }}/lib/python{{ python_version }}/site-packages/tildes.egg-link diff --git a/salt/salt/redis/init.sls b/salt/salt/redis/init.sls deleted file mode 100644 index 8df19b7..0000000 --- a/salt/salt/redis/init.sls +++ /dev/null @@ -1,118 +0,0 @@ -{% set redis_version = '5.0.7' %} - -unpack-redis: - archive.extracted: - - name: /tmp/redis-{{ redis_version }} - - source: - - salt://redis/{{ redis_version }}.tar.gz - - https://github.com/antirez/redis/archive/{{ redis_version }}.tar.gz - - source_hash: sha256=2761422599f8969559e66797cd7f606c16e907bf82d962345a7d366c5d1278df - - unless: /usr/local/bin/redis-server --version | grep v={{ redis_version }} - - options: --strip-components=1 - - enforce_toplevel: False - -install-redis: - pkg.installed: - - pkgs: - - build-essential - cmd.run: - - cwd: /tmp/redis-{{ redis_version }}/ - - names: - - make - - make install - - onchanges: - - archive: unpack-redis - -redis-user: - group.present: - - name: redis - user.present: - - name: redis - - groups: [redis] - - createhome: False - -/run/redis: - file.directory: - - user: redis - - group: redis - - mode: 755 - - require: - - user: redis-user - -/var/lib/redis: - file.directory: - - user: redis - - group: redis - - mode: 700 - - require: - - user: redis-user - -/var/log/redis: - file.directory: - - user: redis - - group: redis - - mode: 744 - - require: - - user: redis-user - -/etc/redis.conf: - file.managed: - - source: salt://redis/redis.conf.jinja2 - - template: jinja - - user: redis - - group: redis - - mode: 600 - - require: - - user: redis-user - -/etc/systemd/system/redis.service: - file.managed: - - source: salt://redis/redis.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: redis.service - -# add the service file for disabling transparent hugepage -/etc/systemd/system/transparent_hugepage.service: - file.managed: - - source: salt://redis/transparent_hugepage.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: disable-transparent-hugepage - -# enable the "disable transparent hugepage" service and run it -disable-transparent-hugepage: - service.enabled: - - name: transparent_hugepage.service - cmd.run: - - name: systemctl start transparent_hugepage.service - - unless: 'cat /sys/kernel/mm/transparent_hugepage/enabled | grep \\[never\\]' - - require_in: - - service: redis.service - - watch_in: - - service: redis.service - -# Set kernel overcommit mode (recommended for Redis) -overcommit-memory: - # will take effect immediately - cmd.run: - - name: sysctl vm.overcommit_memory=1 - - unless: sysctl -n vm.overcommit_memory | grep 1 - - # makes the setting permanent but requires a restart - file.append: - - name: /etc/sysctl.conf - - text: 'vm.overcommit_memory = 1' - -redis.service: - service.running: - - enable: True - - watch: - - file: /etc/redis.conf - - require: - - user: redis-user - - cmd: install-redis diff --git a/salt/salt/redis/modules/redis-cell.sls b/salt/salt/redis/modules/redis-cell.sls deleted file mode 100644 index 6951b32..0000000 --- a/salt/salt/redis/modules/redis-cell.sls +++ /dev/null @@ -1,20 +0,0 @@ -redis-cell-unpack: - archive.extracted: - - name: /opt/redis-cell - - source: - - salt://redis/modules/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz - - https://github.com/brandur/redis-cell/releases/download/v0.2.1/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz - - source_hash: sha256=9427fb100f4cada817f30f854ead7f233de32948a0ec644f15988c275a2ed1cb - - if_missing: /opt/redis-cell - - enforce_toplevel: False - - require_in: - - service: redis.service - -redis-cell-loadmodule-line: - file.accumulated: - - name: redis_loadmodule_lines - - filename: /etc/redis.conf - - text: - - 'loadmodule /opt/redis-cell/libredis_cell.so' - - require_in: - - file: /etc/redis.conf diff --git a/salt/salt/scripts/init.sls b/salt/salt/scripts/init.sls deleted file mode 100644 index 2db61e7..0000000 --- a/salt/salt/scripts/init.sls +++ /dev/null @@ -1,7 +0,0 @@ -/usr/local/bin/activate: - file.managed: - - source: salt://scripts/activate.sh.jinja2 - - template: jinja - - user: root - - group: root - - mode: 755 diff --git a/salt/salt/self-signed-cert.sls b/salt/salt/self-signed-cert.sls deleted file mode 100644 index f1108c1..0000000 --- a/salt/salt/self-signed-cert.sls +++ /dev/null @@ -1,17 +0,0 @@ -self-signed-cert: - pkg.installed: - - name: python3-openssl - module.run: - - tls.create_self_signed_cert: - - days: 3650 - - require: - - pkg: python3-openssl - - unless: ls /etc/pki/tls/certs/localhost.key - file.managed: - - name: /etc/pki/tls/certs/localhost.key - - mode: 600 - - replace: False - - require: - - module: self-signed-cert - - require_in: - - service: nginx diff --git a/salt/salt/sentry/common.jinja2 b/salt/salt/sentry/common.jinja2 deleted file mode 100644 index 055cf3d..0000000 --- a/salt/salt/sentry/common.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ -{% set sentry_cfg_dir = '/etc/sentry' %} - -{% set sentry_venv_dir = '/opt/venvs/sentry' %} -{% set sentry_bin_dir = sentry_venv_dir + '/bin' %} diff --git a/salt/salt/sentry/config.yml.jinja2 b/salt/salt/sentry/config.yml.jinja2 deleted file mode 100644 index d78a880..0000000 --- a/salt/salt/sentry/config.yml.jinja2 +++ /dev/null @@ -1,64 +0,0 @@ -# While a lot of configuration in Sentry can be changed via the UI, for all -# new-style config (as of 8.0) you can also declare values here in this file -# to enforce defaults or to ensure they cannot be changed via the UI. For more -# information see the Sentry documentation. - -############### -# Mail Server # -############### - -mail.backend: 'dummy' # Use dummy if you want to disable email entirely -# mail.host: 'localhost' -# mail.port: 25 -# mail.username: '' -# mail.password: '' -# mail.use-tls: false -# The email address to send on behalf of -# mail.from: 'root@localhost' - -# If you'd like to configure email replies, enable this. -# mail.enable-replies: false - -# When email-replies are enabled, this value is used in the Reply-To header -# mail.reply-hostname: '' - -# If you're using mailgun for inbound mail, set your API key and configure a -# route to forward to /api/hooks/mailgun/inbound/ -# mail.mailgun-api-key: '' - -################### -# System Settings # -################### - -# If this file ever becomes compromised, it's important to regenerate your a new key -# Changing this value will result in all current sessions being invalidated. -# A new key can be generated with `$ sentry config generate-secret-key` -system.secret-key: '{{ pillar['sentry_secret'] }}' - -# The ``redis.clusters`` setting is used, unsurprisingly, to configure Redis -# clusters. These clusters can be then referred to by name when configuring -# backends such as the cache, digests, or TSDB backend. -redis.clusters: - default: - hosts: - 0: - host: 127.0.0.1 - port: 6379 - db: 10 - -################ -# File storage # -################ - -# Uploaded media uses these `filestore` settings. The available -# backends are either `filesystem` or `s3`. - -filestore.backend: 'filesystem' -filestore.options: - location: '/tmp/sentry-files' - -# filestore.backend: 's3' -# filestore.options: -# access_key: 'AKIXXXXXX' -# secret_key: 'XXXXXXX' -# bucket_name: 's3-bucket-name' diff --git a/salt/salt/sentry/init.sls b/salt/salt/sentry/init.sls deleted file mode 100644 index 4c69500..0000000 --- a/salt/salt/sentry/init.sls +++ /dev/null @@ -1,148 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir %} - -sentry-user: - group.present: - - name: sentry - user.present: - - name: sentry - - groups: [sentry] - -build-deps: - pkg.installed: - - pkgs: - - python-setuptools - - python-dev - - python-pip - - python-virtualenv - - libxslt1-dev - - gcc - - libffi-dev - - libjpeg-dev - - libxml2-dev - - libxslt-dev - - libyaml-dev - - libpq-dev - - zlib1g-dev - - postgresql-server-dev-{{ pillar['postgresql_version'] }} - -{{ sentry_venv_dir }}: - virtualenv.managed: - - system_site_packages: False - -pip-install: - pip.installed: - - name: sentry - - bin_env: {{ sentry_venv_dir }} - -pip-install-plugins: - pip.installed: - - name: sentry-plugins - - bin_env: {{ sentry_venv_dir }} - -init-sentry: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry init {{ sentry_cfg_dir }} - - creates: {{ sentry_cfg_dir }} - -postgres-setup: - postgres_user.present: - - name: sentry - postgres_database.present: - - name: sentry - - owner: sentry - # sentry migrations should add this, but it fails due to not being superuser - postgres_extension.present: - - name: citext - - maintenance_db: sentry - -{{ sentry_cfg_dir }}/sentry.conf.py: - file.managed: - - source: salt://sentry/sentry.conf.py - -{{ sentry_cfg_dir }}/config.yml: - file.managed: - - source: salt://sentry/config.yml.jinja2 - - template: jinja - -update-pg_hba: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: 'host sameuser sentry 127.0.0.1/32 trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -create-sentry-db: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry upgrade --noinput - - env: - - SENTRY_CONF: '{{ sentry_cfg_dir }}' - - onchanges: - - cmd: init-sentry - -create-sentry-user: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry createuser --no-input --email {{ pillar['sentry_email'] }} --password {{ pillar['sentry_password'] }} - - env: - - SENTRY_CONF: '{{ sentry_cfg_dir }}' - - onchanges: - - cmd: init-sentry - -/etc/systemd/system/sentry-web.service: - file.managed: - - source: salt://sentry/sentry-web.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/sentry-worker.service: - file.managed: - - source: salt://sentry/sentry-worker.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/sentry-cron.service: - file.managed: - - source: salt://sentry/sentry-cron.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -sentry-web: - service.running: - - enable: True - -sentry-worker: - service.running: - - enable: True - -sentry-cron: - service.running: - - enable: True - -sentry-cleanup: - cron.present: - - name: {{ sentry_bin_dir }}/sentry cleanup --days=30 - - hour: 4 - - minute: 0 - -/etc/nginx/sites-available/sentry.conf: - file.managed: - - source: salt://sentry/sentry.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/sentry.conf: - file.symlink: - - target: /etc/nginx/sites-available/sentry.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/sentry/sentry-cron.service.jinja2 b/salt/salt/sentry/sentry-cron.service.jinja2 deleted file mode 100644 index c533422..0000000 --- a/salt/salt/sentry/sentry-cron.service.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Beat Service -After=network.target - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run cron - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry-web.service.jinja2 b/salt/salt/sentry/sentry-web.service.jinja2 deleted file mode 100644 index d140c28..0000000 --- a/salt/salt/sentry/sentry-web.service.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Main Service -After=network.target -Requires=sentry-worker.service -Requires=sentry-cron.service - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run web - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry-worker.service.jinja2 b/salt/salt/sentry/sentry-worker.service.jinja2 deleted file mode 100644 index 42d9373..0000000 --- a/salt/salt/sentry/sentry-worker.service.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Background Worker -After=network.target - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run worker - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry.conf.jinja2 b/salt/salt/sentry/sentry.conf.jinja2 deleted file mode 100644 index 96ca7a5..0000000 --- a/salt/salt/sentry/sentry.conf.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['sentry_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - proxy_pass http://localhost:9000; - } -} diff --git a/salt/salt/sentry/sentry.conf.py b/salt/salt/sentry/sentry.conf.py deleted file mode 100644 index 6e3459d..0000000 --- a/salt/salt/sentry/sentry.conf.py +++ /dev/null @@ -1,135 +0,0 @@ -# This file is just Python, with a touch of Django which means -# you can inherit and tweak settings to your hearts content. -from sentry.conf.server import * - -import os.path - -CONF_ROOT = os.path.dirname(__file__) - -DATABASES = { - 'default': { - 'ENGINE': 'sentry.db.postgres', - 'NAME': 'sentry', - 'USER': 'sentry', - 'PASSWORD': '', - 'HOST': '127.0.0.1', - 'PORT': '', - 'AUTOCOMMIT': True, - 'ATOMIC_REQUESTS': False, - } -} - -# You should not change this setting after your database has been created -# unless you have altered all schemas first -SENTRY_USE_BIG_INTS = True - -# If you're expecting any kind of real traffic on Sentry, we highly recommend -# configuring the CACHES and Redis settings - -########### -# General # -########### - -# Instruct Sentry that this install intends to be run by a single organization -# and thus various UI optimizations should be enabled. -SENTRY_SINGLE_ORGANIZATION = True -DEBUG = False - -######### -# Cache # -######### - -# Sentry currently utilizes two separate mechanisms. While CACHES is not a -# requirement, it will optimize several high throughput patterns. - -# If you wish to use memcached, install the dependencies and adjust the config -# as shown: -# -# pip install python-memcached -# -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', -# 'LOCATION': ['127.0.0.1:11211'], -# } -# } - -# A primary cache is required for things such as processing events -SENTRY_CACHE = 'sentry.cache.redis.RedisCache' - -######### -# Queue # -######### - -# See https://docs.sentry.io/on-premise/server/queue/ for more -# information on configuring your queue broker and workers. Sentry relies -# on a Python framework called Celery to manage queues. - -BROKER_URL = 'redis://localhost:6379' - -############### -# Rate Limits # -############### - -# Rate limits apply to notification handlers and are enforced per-project -# automatically. - -SENTRY_RATELIMITER = 'sentry.ratelimits.redis.RedisRateLimiter' - -################## -# Update Buffers # -################## - -# Buffers (combined with queueing) act as an intermediate layer between the -# database and the storage API. They will greatly improve efficiency on large -# numbers of the same events being sent to the API in a short amount of time. -# (read: if you send any kind of real data to Sentry, you should enable buffers) - -SENTRY_BUFFER = 'sentry.buffer.redis.RedisBuffer' - -########## -# Quotas # -########## - -# Quotas allow you to rate limit individual projects or the Sentry install as -# a whole. - -SENTRY_QUOTAS = 'sentry.quotas.redis.RedisQuota' - -######## -# TSDB # -######## - -# The TSDB is used for building charts as well as making things like per-rate -# alerts possible. - -SENTRY_TSDB = 'sentry.tsdb.redis.RedisTSDB' - -########### -# Digests # -########### - -# The digest backend powers notification summaries. - -SENTRY_DIGESTS = 'sentry.digests.backends.redis.RedisBackend' - -############## -# Web Server # -############## - -# If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto -# header and uncomment the following settings -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -SESSION_COOKIE_SECURE = True -CSRF_COOKIE_SECURE = True - -# If you're not hosting at the root of your web server, -# you need to uncomment and set it to the path where Sentry is hosted. -#FORCE_SCRIPT_NAME = '/sentry' - -SENTRY_WEB_HOST = '0.0.0.0' -SENTRY_WEB_PORT = 9000 -SENTRY_WEB_OPTIONS = { - # 'workers': 3, # the number of web workers - # 'protocol': 'uwsgi', # Enable uwsgi protocol instead of http -} diff --git a/salt/salt/tildes-wiki.sls b/salt/salt/tildes-wiki.sls deleted file mode 100644 index a6c2478..0000000 --- a/salt/salt/tildes-wiki.sls +++ /dev/null @@ -1,23 +0,0 @@ -{% from 'common.jinja2' import app_username %} - -# Create the base directory for the wiki files and initialize a git repo -/var/lib/tildes-wiki: - file.directory: - - user: {{ app_username }} - - group: {{ app_username }} - - mode: 775 - git.present: - - bare: False - - user: {{ app_username }} - -# Create the initial (empty) commit in the new git repo -wiki-initial-commit: - cmd.run: - - names: - - git config user.name "Tildes" - - git config user.email "Tildes" - - git commit --allow-empty -m "Initial commit" - - cwd: /var/lib/tildes-wiki - - runas: {{ app_username }} - - onchanges: - - file: /var/lib/tildes-wiki diff --git a/salt/salt/top.sls b/salt/salt/top.sls deleted file mode 100644 index 9ef97a7..0000000 --- a/salt/salt/top.sls +++ /dev/null @@ -1,46 +0,0 @@ -base: - 'dev or prod': - - general-config - - gunicorn - - nginx - - nginx.site-config - - postgresql - - postgresql.site-db - - postgresql.pgbouncer - - python - - redis - - redis.modules.redis-cell - - postgresql-redis-bridge - - scripts - - cmark-gfm - - prometheus.exporters.node_exporter - - prometheus.exporters.postgres_exporter - - prometheus.exporters.redis_exporter - - consumers - - tildes-wiki - - boussole - - webassets - - pts-lbsearch - - cronjobs - - final-setup # keep this state file last - 'dev': - - postgresql.test-db - - self-signed-cert - - development - - prometheus - - nodejs - - java - 'prod': - - nginx.shortener-config - - nginx.static-sites-config - - postgresql.prod-config - - prod-config - 'monitoring': - - nginx - - self-signed-cert - - postgresql - - redis - - sentry - - grafana - - prometheus - - prometheus.exporters.blackbox_exporter diff --git a/salt/salt/webassets.sls b/salt/salt/webassets.sls deleted file mode 100644 index f2de67b..0000000 --- a/salt/salt/webassets.sls +++ /dev/null @@ -1,25 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username %} - -# webassets will crash the site unless this file exists, make sure it's always there -{{ app_dir }}/static/css/site-icons.css: - file.managed: - - user: {{ app_username }} - - group: {{ app_username }} - - create: True - - replace: False - -/etc/systemd/system/webassets.service: - file.managed: - - source: salt://webassets.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: webassets.service - -webassets.service: - service.running: - - enable: True - - require: - - pip: pip-installs