diff --git a/Vagrantfile b/Vagrantfile index d7e9e75..dcff88d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,25 +4,39 @@ VAGRANT_CONFIG_VERSION = "2" Vagrant.configure(VAGRANT_CONFIG_VERSION) do |config| - config.vm.box = "ubuntu/xenial64" + # Using the "contrib" version for vboxsf module for synced folders + config.vm.box = "debian/contrib-buster64" # Main application folder config.vm.synced_folder "tildes/", "/opt/tildes/" - # Mount the salt file root and pillar root - config.vm.synced_folder "salt/salt/", "/srv/salt/" - config.vm.synced_folder "salt/pillar/", "/srv/pillar/" + config.vm.synced_folder "ansible/", "/srv/ansible" config.vm.network "forwarded_port", guest: 443, host: 4443 config.vm.network "forwarded_port", guest: 9090, host: 9090 - # Masterless salt provisioning - config.vm.provision :salt do |salt| - salt.masterless = true - salt.minion_config = "salt/minion" - salt.run_highstate = true - salt.verbose = true - salt.log_level = "info" + config.vm.provision "ansible_local" do |ansible| + ansible.install_mode = "pip" + + # Since Debian Buster still uses Python 2.7 by default and the pip bootstrap + # script is no longer compatible with 2.7, we need to specify the installation + # command manually. If we upgrade to a newer version of Debian that defaults to + # Python 3.6+, this should no longer be necessary. + ansible.pip_install_cmd = "sudo apt-get install -y python3-distutils && curl -s https://bootstrap.pypa.io/get-pip.py | sudo python3" + + # Vagrant doesn't currently recognize the new format for Ansible versions + # (e.g. "ansible [core 2.11.1]"), so the compatibility mode is set incorrectly. + # A new version of Vagrant should resolve this soon. + ansible.compatibility_mode = "2.0" + + # put the VM into the "dev" and "app_server" Ansible groups + ansible.groups = { + "dev" => ["default"], + "app_server" => ["default"], + } + + ansible.galaxy_role_file = "ansible/requirements.yml" + ansible.playbook = "ansible/playbook.yml" end config.vm.provider "virtualbox" do |vb| diff --git a/ansible/common.yml b/ansible/common.yml new file mode 100644 index 0000000..9797079 --- /dev/null +++ b/ansible/common.yml @@ -0,0 +1,9 @@ +--- +- hosts: all + become: true + + vars_files: + - vars.yml + + roles: + - common diff --git a/ansible/group_vars/dev.yml b/ansible/group_vars/dev.yml new file mode 100644 index 0000000..a265d20 --- /dev/null +++ b/ansible/group_vars/dev.yml @@ -0,0 +1,33 @@ +--- +app_username: vagrant +pip_requirements_filename: requirements-dev.txt +ini_file: development.ini + +gunicorn_args: --reload + +# have to disable sendfile for vagrant due to a virtualbox bug +nginx_enable_sendfile: false +nginx_worker_processes: 1 + +nginx_enable_hsts: false +nginx_enable_csp: false + +postgresql_tildes_databases: + - tildes + - tildes_test +postgresql_tildes_user_flags: "SUPERUSER" +tildes_database_insert_dev_data: true + +hsts_max_age: 60 + +site_hostname: localhost + +ssl_cert_dir: /etc/pki/tls/certs +ssl_cert_path: "{{ ssl_cert_dir }}/localhost.crt" +ssl_private_key_path: "{{ ssl_cert_dir }}/localhost.key" + +ansible_python_interpreter: /usr/bin/python3 + +# Workaround for some Ansible permissions issues when becoming an unprivileged user +# (this has some risks, but should be fine for our use) +ansible_shell_allow_world_readable_temp: true diff --git a/ansible/group_vars/prod.yml b/ansible/group_vars/prod.yml new file mode 100644 index 0000000..b405874 --- /dev/null +++ b/ansible/group_vars/prod.yml @@ -0,0 +1,52 @@ +--- +app_username: tildes +postgresql_settings: + checkpoint_completion_target: 0.7 + default_statistics_target: 100 + effective_cache_size: 24GB + effective_io_concurrency: 200 + maintenance_work_mem: 2GB + max_parallel_workers: 8 + max_parallel_workers_per_gather: 4 + max_wal_size: 2GB + max_worker_processes: 8 + min_wal_size: 1GB + random_page_cost: 1.1 + shared_buffers: 8GB + wal_buffers: 16MB + work_mem: 10485kB +ini_file: production.ini +gunicorn_args: --workers 8 +site_hostname: tildes.net + +# add extra prod-only consumer services (e.g. ones that use external APIs) +consumers: + - comment_user_mentions_generator + - post_processing_script_runner + - site_icon_downloader + - topic_embedly_extractor + - topic_interesting_activity_updater + - topic_metadata_generator + - topic_youtube_scraper + +ipv6_device: eno1 +ipv6_address: "2607:5300:0203:2e7a::" +ipv6_gateway: "2607:5300:0203:2eff:ff:ff:ff:ff" + +prometheus_ips: + - "2607:5300:201:3100::6e77" + +prometheus_consumer_scrape_targets: + comment_user_mentions_generator: 25010 + post_processing_script_runner: 25016 + site_icon_downloader: 25011 + topic_embedly_extractor: 25012 + topic_interesting_activity_updater: 25013 + topic_metadata_generator: 25014 + topic_youtube_scraper: 25015 + +ansible_python_interpreter: /usr/bin/python3 + +# Workaround for some Ansible permissions issues when becoming an unprivileged user +# (this has some risks, but should be fine for our use) +ansible_shell_allow_world_readable_temp: true diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 0000000..86f5dad --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,54 @@ +--- +- hosts: all + become: true + vars_files: + - vars.yml + roles: + - common + +- hosts: app_server + become: true + vars_files: + - vars.yml + roles: + - cmark-gfm + - pts_lbsearch + - python + - gunicorn + - nginx + - nginx_site_config + - postgresql + - postgresql_plpython + - postgresql_tildes_dbs + - pgbouncer + - redis + - redis_module_cell + - postgresql_redis_bridge + - boussole + - webassets + - scripts + - prometheus_node_exporter + - prometheus_postgres_exporter + - prometheus_redis_exporter + - consumers + - cronjobs + - wiki_repo + +- hosts: dev + become: true + vars_files: + - vars.yml + roles: + - self_signed_ssl_cert + - prometheus + - java + - nodejs + - development + +- hosts: prod + become: true + vars_files: + - vars.yml + roles: + - nginx_prod_config + - ipv6_networking diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..8dd5161 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - community.general diff --git a/ansible/roles/boussole/defaults/main.yml b/ansible/roles/boussole/defaults/main.yml new file mode 100644 index 0000000..f7173fb --- /dev/null +++ b/ansible/roles/boussole/defaults/main.yml @@ -0,0 +1,2 @@ +--- +boussole_venv_dir: /opt/venvs/boussole diff --git a/ansible/roles/boussole/tasks/main.yml b/ansible/roles/boussole/tasks/main.yml new file mode 100644 index 0000000..103264c --- /dev/null +++ b/ansible/roles/boussole/tasks/main.yml @@ -0,0 +1,44 @@ +--- +- name: Create venv and install boussole with pip + pip: + virtualenv: "{{ boussole_venv_dir }}" + virtualenv_command: python{{ python_version }} -m venv + name: boussole + +- name: Create systemd service file + template: + src: boussole.service.jinja2 + dest: /etc/systemd/system/boussole.service + owner: root + group: root + mode: 0644 + +- name: Start and enable boussole service + service: + name: boussole + state: started + enabled: true + +- name: Create directory for compiled CSS + file: + path: "{{ app_dir }}/static/css" + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0755 + +- name: Check if any compiled CSS files exist + find: + path: "{{ app_dir }}/static/css" + patterns: + - "*.css" + register: compiled_css_files + +- name: Run boussole manually if no compiled CSS exists yet + command: + chdir: "{{ app_dir }}" + cmd: "{{ boussole_venv_dir }}/bin/boussole compile --backend=yaml --config=boussole.yaml" + environment: + LC_ALL: C.UTF-8 + LANG: C.UTF-8 + when: compiled_css_files.matched == 0 diff --git a/salt/salt/boussole.service.jinja2 b/ansible/roles/boussole/templates/boussole.service.jinja2 similarity index 63% rename from salt/salt/boussole.service.jinja2 rename to ansible/roles/boussole/templates/boussole.service.jinja2 index 05c418e..064066d 100644 --- a/salt/salt/boussole.service.jinja2 +++ b/ansible/roles/boussole/templates/boussole.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username -%} [Unit] Description=Boussole - auto-compile SCSS files on change @@ -7,7 +6,7 @@ User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }} Environment="LC_ALL=C.UTF-8" "LANG=C.UTF-8" -ExecStart=/opt/venvs/boussole/bin/boussole watch --backend=yaml --config=boussole.yaml --poll +ExecStart={{ boussole_venv_dir }}/bin/boussole watch --backend=yaml --config=boussole.yaml --poll Restart=always RestartSec=5 diff --git a/ansible/roles/cmark-gfm/tasks/main.yml b/ansible/roles/cmark-gfm/tasks/main.yml new file mode 100644 index 0000000..3111153 --- /dev/null +++ b/ansible/roles/cmark-gfm/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Check if cmark-gfm is installed + stat: + path: /usr/local/lib/libcmark-gfm.so + register: cmark_gfm_library + +- name: Download and install cmark-gfm + when: not cmark_gfm_library.stat.exists + block: + - name: Download cmark-gfm from GitHub + get_url: + dest: /tmp/cmark-gfm.tar.gz + url: https://github.com/github/cmark-gfm/archive/0.29.0.gfm.0.tar.gz + checksum: sha256:6a94aeaa59a583fadcbf28de81dea8641b3f56d935dda5b2447a3c8df6c95fea + + - name: Create temp directory to extract cmark-gfm to + file: + path: /tmp/cmark-gfm + state: directory + + - name: Extract cmark-gfm + unarchive: + remote_src: true + src: /tmp/cmark-gfm.tar.gz + dest: /tmp/cmark-gfm + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for cmark-gfm + apt: + name: + - build-essential + - cmake + + - name: Install cmark-gfm + shell: + chdir: /tmp/cmark-gfm + cmd: | + make + make install diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000..a5e4aa4 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Set time zone to UTC + community.general.timezone: + name: Etc/UTC + +- name: Create group for app user + group: + name: "{{ app_username }}" + +- name: Create app user + user: + name: "{{ app_username }}" + group: "{{ app_username }}" diff --git a/ansible/roles/consumers/defaults/main.yml b/ansible/roles/consumers/defaults/main.yml new file mode 100644 index 0000000..d0db61d --- /dev/null +++ b/ansible/roles/consumers/defaults/main.yml @@ -0,0 +1,6 @@ +--- +consumers: + - comment_user_mentions_generator + - post_processing_script_runner + - topic_interesting_activity_updater + - topic_metadata_generator diff --git a/ansible/roles/consumers/meta/main.yml b/ansible/roles/consumers/meta/main.yml new file mode 100644 index 0000000..0a45cb3 --- /dev/null +++ b/ansible/roles/consumers/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: python + - role: redis diff --git a/ansible/roles/consumers/tasks/main.yml b/ansible/roles/consumers/tasks/main.yml new file mode 100644 index 0000000..ded3628 --- /dev/null +++ b/ansible/roles/consumers/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: Set up service files for background consumers + template: + src: "consumer.service.jinja2" + dest: /etc/systemd/system/consumer-{{ item }}.service + owner: root + group: root + mode: 0644 + loop: "{{ consumers }}" + +- name: Start and enable all consumer services + service: + name: consumer-{{ item }} + state: started + enabled: true + loop: "{{ consumers }}" diff --git a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 b/ansible/roles/consumers/templates/consumer.service.jinja2 similarity index 50% rename from salt/salt/consumers/topic_youtube_scraper.service.jinja2 rename to ansible/roles/consumers/templates/consumer.service.jinja2 index 8623e5e..4e74337 100644 --- a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 +++ b/ansible/roles/consumers/templates/consumer.service.jinja2 @@ -1,6 +1,5 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] -Description=Topic Youtube Scraper (Queue Consumer) +Description={{ item.replace("_", " ").title() }} (Queue Consumer) Requires=redis.service After=redis.service PartOf=redis.service @@ -9,8 +8,8 @@ PartOf=redis.service User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_youtube_scraper.py +Environment="INI_FILE={{ app_dir }}/{{ ini_file }}" +ExecStart={{ bin_dir }}/python {{ item }}.py Restart=always RestartSec=5 diff --git a/ansible/roles/cronjobs/tasks/main.yml b/ansible/roles/cronjobs/tasks/main.yml new file mode 100644 index 0000000..73c58df --- /dev/null +++ b/ansible/roles/cronjobs/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: Add cronjob for lifting expired temporary bans + cron: + name: lift_expired_temporary_bans + job: "{{ bin_dir }}/python -c \"from scripts.lift_expired_temporary_bans import lift_expired_temporary_bans; lift_expired_temporary_bans('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 1 + +- name: Add cronjob for closing voting on old posts + cron: + name: close_voting_on_old_posts + job: "{{ bin_dir }}/python -c \"from scripts.close_voting_on_old_posts import close_voting_on_old_posts; close_voting_on_old_posts('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 3 + +- name: Add cronjob for cleaning up private data + cron: + name: clean_private_data + job: "{{ bin_dir }}/python -c \"from scripts.clean_private_data import clean_all_data; clean_all_data('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: 4 + minute: 10 + +- name: Add cronjob for generating yesterday's group stats + cron: + name: generate_group_stats_for_yesterday + job: "{{ bin_dir }}/python -c \"from scripts.generate_group_stats_for_yesterday import generate_stats; generate_stats('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: 0 + minute: 10 + +- name: Add cronjob for generating site-icons CSS file + cron: + name: generate_site_icons_css + job: "{{ bin_dir }}/python -c \"from scripts.generate_site_icons_css import generate_css; generate_css()\"" + user: "{{ app_username }}" + minute: "*/5" + +- name: Add cronjob for posting scheduled topics + cron: + name: post_scheduled_topics + job: "{{ bin_dir }}/python -c \"from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + +- name: Add cronjob for updating all groups' lists of common topic tags + cron: + name: update_groups_common_topic_tags + job: "{{ bin_dir }}/python -c \"from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ ini_file }}')\"" + user: "{{ app_username }}" + hour: "*" + minute: 0 diff --git a/ansible/roles/development/files/ipython_config.py b/ansible/roles/development/files/ipython_config.py new file mode 100644 index 0000000..bcad1ad --- /dev/null +++ b/ansible/roles/development/files/ipython_config.py @@ -0,0 +1,2 @@ +c.InteractiveShellApp.extensions = ['autoreload'] +c.InteractiveShellApp.exec_lines = ['%autoreload 2'] diff --git a/ansible/roles/development/tasks/main.yml b/ansible/roles/development/tasks/main.yml new file mode 100644 index 0000000..4704116 --- /dev/null +++ b/ansible/roles/development/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Create IPython profile + become_user: "{{ app_username }}" + command: + cmd: "{{ bin_dir }}/ipython profile create" + creates: /home/{{ app_username }}/.ipython/profile_default + +- name: Create IPython config file + copy: + src: ipython_config.py + dest: /home/{{ app_username }}/.ipython/profile_default/ipython_config.py + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0744 + +- name: Automatically activate venv on login and in new shells + lineinfile: + path: /home/{{ app_username }}/.bashrc + line: source activate + owner: "{{ app_username }}" + group: "{{ app_username }}" + +- name: Add invoke's tab-completion script to support completing invoke task names + lineinfile: + path: /home/{{ app_username }}/.bashrc + line: source <(invoke --print-completion-script bash) diff --git a/salt/salt/gunicorn/gunicorn_reloader.path b/ansible/roles/gunicorn/files/gunicorn_reloader.path similarity index 100% rename from salt/salt/gunicorn/gunicorn_reloader.path rename to ansible/roles/gunicorn/files/gunicorn_reloader.path diff --git a/salt/salt/gunicorn/gunicorn_reloader.service b/ansible/roles/gunicorn/files/gunicorn_reloader.service similarity index 100% rename from salt/salt/gunicorn/gunicorn_reloader.service rename to ansible/roles/gunicorn/files/gunicorn_reloader.service diff --git a/ansible/roles/gunicorn/meta/main.yml b/ansible/roles/gunicorn/meta/main.yml new file mode 100644 index 0000000..f7230ad --- /dev/null +++ b/ansible/roles/gunicorn/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: python diff --git a/ansible/roles/gunicorn/tasks/main.yml b/ansible/roles/gunicorn/tasks/main.yml new file mode 100644 index 0000000..d541fde --- /dev/null +++ b/ansible/roles/gunicorn/tasks/main.yml @@ -0,0 +1,54 @@ +--- +- name: Create gunicorn service file + template: + src: gunicorn.service.jinja2 + dest: /etc/systemd/system/gunicorn.service + owner: root + group: root + mode: 0644 + +- name: Create gunicorn socket file + template: + src: gunicorn.socket.jinja2 + dest: /etc/systemd/system/gunicorn.socket + owner: root + group: root + mode: 0644 + +- name: Create gunicorn tmpfiles.d configuration file + template: + src: gunicorn.conf.jinja2 + dest: /usr/lib/tmpfiles.d/gunicorn.conf + owner: root + group: root + mode: 0644 + +- name: Start and enable gunicorn.socket service + service: + name: gunicorn.socket + state: started + enabled: true + +# Set up the gunicorn_reloader service, which reloads gunicorn whenever certain files +# are changed (such as static files, to update the cache-busting strings) +- name: Create gunicorn_reloader service file + copy: + src: gunicorn_reloader.service + dest: /etc/systemd/system/gunicorn_reloader.service + owner: root + group: root + mode: 0644 + +- name: Create gunicorn_reloader path-monitoring file + copy: + src: gunicorn_reloader.path + dest: /etc/systemd/system/gunicorn_reloader.path + owner: root + group: root + mode: 0644 + +- name: Start and enable gunicorn_reloader path-monitoring service + service: + name: gunicorn_reloader.path + state: started + enabled: true diff --git a/salt/salt/gunicorn/gunicorn.conf.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 similarity index 56% rename from salt/salt/gunicorn/gunicorn.conf.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 index 7554bc1..e1c3af6 100644 --- a/salt/salt/gunicorn/gunicorn.conf.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.conf.jinja2 @@ -1,2 +1 @@ -{% from 'common.jinja2' import app_username %} d /run/gunicorn 0755 {{ app_username }} {{ app_username }} - diff --git a/salt/salt/gunicorn/gunicorn.service.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.service.jinja2 similarity index 61% rename from salt/salt/gunicorn/gunicorn.service.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.service.jinja2 index 3fd06f5..fbf9912 100644 --- a/salt/salt/gunicorn/gunicorn.service.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=gunicorn daemon Requires=gunicorn.socket @@ -10,7 +9,7 @@ User={{ app_username }} Group={{ app_username }} RuntimeDirectory=gunicorn WorkingDirectory={{ app_dir }} -ExecStart={{ bin_dir }}/gunicorn --paste {{ pillar['ini_file'] }} --config {{ app_dir }}/gunicorn_config.py --bind unix:/run/gunicorn/socket --pid /run/gunicorn/pid {{ pillar['gunicorn_args'] }} +ExecStart={{ bin_dir }}/gunicorn --paste {{ ini_file }} --config {{ app_dir }}/gunicorn_config.py --bind unix:/run/gunicorn/socket --pid /run/gunicorn/pid {{ gunicorn_args }} ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true diff --git a/salt/salt/gunicorn/gunicorn.socket.jinja2 b/ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 similarity index 80% rename from salt/salt/gunicorn/gunicorn.socket.jinja2 rename to ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 index a81da97..3773779 100644 --- a/salt/salt/gunicorn/gunicorn.socket.jinja2 +++ b/ansible/roles/gunicorn/templates/gunicorn.socket.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_username -%} [Unit] Description=gunicorn socket PartOf=gunicorn.service diff --git a/ansible/roles/ipv6_networking/defaults/main.yml b/ansible/roles/ipv6_networking/defaults/main.yml new file mode 100644 index 0000000..99539e9 --- /dev/null +++ b/ansible/roles/ipv6_networking/defaults/main.yml @@ -0,0 +1,2 @@ +--- +ipv6_device: eth0 diff --git a/ansible/roles/ipv6_networking/tasks/main.yml b/ansible/roles/ipv6_networking/tasks/main.yml new file mode 100644 index 0000000..8a8d847 --- /dev/null +++ b/ansible/roles/ipv6_networking/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Enable IPv6 networking + blockinfile: + path: /etc/network/interfaces + block: | + iface {{ ipv6_device }} inet6 static + address {{ ipv6_address }} + netmask 64 + + post-up sleep 5; /sbin/ip -family inet6 route add {{ ipv6_gateway }} dev {{ ipv6_device }} + post-up sleep 5; /sbin/ip -family inet6 route add default via {{ ipv6_gateway }} + pre-down /sbin/ip -family inet6 route del default via {{ ipv6_gateway }} + pre-down /sbin/ip -family inet6 route del {{ ipv6_gateway }} dev {{ ipv6_device }} + +# apt seems to hang a lot when using IPv6 +- name: Force apt not to use IPv6 + lineinfile: + path: /etc/apt/apt.conf.d/99force-ipv4 + line: Acquire::ForceIPv4 "true"; + create: true + owner: root + group: root + mode: 0644 diff --git a/ansible/roles/java/tasks/main.yml b/ansible/roles/java/tasks/main.yml new file mode 100644 index 0000000..dfa1e0b --- /dev/null +++ b/ansible/roles/java/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: Install OpenJDK Java runtime + apt: + name: openjdk-11-jre diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml new file mode 100644 index 0000000..eacc87f --- /dev/null +++ b/ansible/roles/nginx/defaults/main.yml @@ -0,0 +1,6 @@ +--- +nginx_enable_sendfile: true +nginx_worker_processes: auto + +ssl_cert_path: /etc/letsencrypt/live/{{ site_hostname }}/fullchain.pem +ssl_private_key_path: /etc/letsencrypt/live/{{ site_hostname }}/privkey.pem diff --git a/salt/salt/nginx/logrotate b/ansible/roles/nginx/files/logrotate similarity index 86% rename from salt/salt/nginx/logrotate rename to ansible/roles/nginx/files/logrotate index 9a4fc89..a089752 100644 --- a/salt/salt/nginx/logrotate +++ b/ansible/roles/nginx/files/logrotate @@ -1,3 +1,4 @@ +# rotate nginx log files daily and delete after 30 days /var/log/nginx/*.log { daily missingok diff --git a/ansible/roles/nginx/handlers/main.yml b/ansible/roles/nginx/handlers/main.yml new file mode 100644 index 0000000..eb0e7f1 --- /dev/null +++ b/ansible/roles/nginx/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/nginx/tasks/main.yml b/ansible/roles/nginx/tasks/main.yml new file mode 100644 index 0000000..467449d --- /dev/null +++ b/ansible/roles/nginx/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: Add APT key for nginx repository + apt_key: + url: https://nginx.org/keys/nginx_signing.key + +- name: Add nginx APT repository + apt_repository: + repo: deb http://nginx.org/packages/debian/ buster nginx + +- name: Install nginx + apt: + name: nginx + +- name: Start and enable nginx service + service: + name: nginx + state: started + enabled: true + +- name: Create nginx.conf file + template: + src: nginx.conf.jinja2 + dest: /etc/nginx/nginx.conf + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Create sites-available directory + file: + path: /etc/nginx/sites-available + state: directory + owner: root + group: root + mode: 0755 + +- name: Create sites-enabled directory + file: + path: /etc/nginx/sites-enabled + state: directory + owner: root + group: root + mode: 0744 + +- name: Add logrotate config + copy: + src: logrotate + dest: /etc/logrotate.d/nginx + owner: root + group: root + mode: 0644 diff --git a/salt/salt/nginx/nginx.conf.jinja2 b/ansible/roles/nginx/templates/nginx.conf.jinja2 similarity index 89% rename from salt/salt/nginx/nginx.conf.jinja2 rename to ansible/roles/nginx/templates/nginx.conf.jinja2 index 493fd01..af1efc5 100644 --- a/salt/salt/nginx/nginx.conf.jinja2 +++ b/ansible/roles/nginx/templates/nginx.conf.jinja2 @@ -1,5 +1,5 @@ user nginx; -worker_processes {{ pillar['nginx_worker_processes'] }}; +worker_processes {{ nginx_worker_processes }}; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; @@ -22,11 +22,10 @@ http { access_log /var/log/nginx/access.log main; - {% if grains['id'] == 'dev' %} - # have to disable sendfile for vagrant due to a virtualbox bug - sendfile off; - {% else %} + {% if nginx_enable_sendfile %} sendfile on; + {% else %} + sendfile off; {% endif %} # define a rate-limiting zone to use, and return HTTP 429 if exceeded @@ -77,8 +76,8 @@ http { text/x-component text/x-cross-domain-policy; - ssl_certificate {{ pillar['ssl_cert_path'] }}; - ssl_certificate_key {{ pillar['ssl_private_key_path'] }}; + ssl_certificate {{ ssl_cert_path }}; + ssl_certificate_key {{ ssl_private_key_path }}; ssl_session_timeout 1d; ssl_session_cache shared:SSL:50m; ssl_session_tickets off; diff --git a/ansible/roles/nginx_prod_config/defaults/main.yml b/ansible/roles/nginx_prod_config/defaults/main.yml new file mode 100644 index 0000000..fbb06da --- /dev/null +++ b/ansible/roles/nginx_prod_config/defaults/main.yml @@ -0,0 +1,3 @@ +--- +hsts_max_age: 63072000 +static_sites_dir: /opt/tildes-static-sites diff --git a/ansible/roles/nginx_prod_config/meta/main.yml b/ansible/roles/nginx_prod_config/meta/main.yml new file mode 100644 index 0000000..80b68ef --- /dev/null +++ b/ansible/roles/nginx_prod_config/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: nginx + - role: gunicorn diff --git a/ansible/roles/nginx_prod_config/tasks/main.yml b/ansible/roles/nginx_prod_config/tasks/main.yml new file mode 100644 index 0000000..bfc3b0e --- /dev/null +++ b/ansible/roles/nginx_prod_config/tasks/main.yml @@ -0,0 +1,38 @@ +--- +- name: Add shortener config file + template: + src: tildes-shortener.conf.jinja2 + dest: /etc/nginx/sites-available/tildes-shortener.conf + owner: root + group: root + mode: 0644 + +- name: Enable shortener in nginx + file: + path: /etc/nginx/sites-enabled/tildes-shortener.conf + src: /etc/nginx/sites-available/tildes-shortener.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Add static sites config file + template: + src: tildes-static-sites.conf.jinja2 + dest: /etc/nginx/sites-available/tildes-static-sites.conf + owner: root + group: root + mode: 0644 + +- name: Enable static sites in nginx + file: + path: /etc/nginx/sites-enabled/tildes-static-sites.conf + src: /etc/nginx/sites-available/tildes-static-sites.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx diff --git a/salt/salt/nginx/tildes-shortener.conf.jinja2 b/ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 similarity index 84% rename from salt/salt/nginx/tildes-shortener.conf.jinja2 rename to ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 index 019f3fd..9ef29d1 100644 --- a/salt/salt/nginx/tildes-shortener.conf.jinja2 +++ b/ansible/roles/nginx_prod_config/templates/tildes-shortener.conf.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir -%} server { listen 443 ssl http2; listen [::]:443 ssl http2; @@ -7,7 +6,7 @@ server { keepalive_timeout 5; - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; # Are these security headers unnecessary when we're just redirecting? add_header X-Content-Type-Options "nosniff" always; diff --git a/salt/salt/nginx/tildes-static-sites.conf.jinja2 b/ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 similarity index 91% rename from salt/salt/nginx/tildes-static-sites.conf.jinja2 rename to ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 index 869bcf6..291dc71 100644 --- a/salt/salt/nginx/tildes-static-sites.conf.jinja2 +++ b/ansible/roles/nginx_prod_config/templates/tildes-static-sites.conf.jinja2 @@ -1,11 +1,9 @@ -{% from 'common.jinja2' import static_sites_dir -%} - {% for subdomain in ('blog', 'docs') %} server { listen 443 ssl http2; listen [::]:443 ssl http2; - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; add_header X-Content-Type-Options "nosniff" always; add_header X-Frame-Options "SAMEORIGIN" always; diff --git a/ansible/roles/nginx_site_config/defaults/main.yml b/ansible/roles/nginx_site_config/defaults/main.yml new file mode 100644 index 0000000..7a51abd --- /dev/null +++ b/ansible/roles/nginx_site_config/defaults/main.yml @@ -0,0 +1,11 @@ +--- +nginx_enable_hsts: true +nginx_enable_csp: true +nginx_enable_ratelimiting: true +nginx_redirect_www: true + +prometheus_ips: + - 127.0.0.1 + - ::1 + +hsts_max_age: 63072000 diff --git a/ansible/roles/nginx_site_config/meta/main.yml b/ansible/roles/nginx_site_config/meta/main.yml new file mode 100644 index 0000000..80b68ef --- /dev/null +++ b/ansible/roles/nginx_site_config/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: nginx + - role: gunicorn diff --git a/ansible/roles/nginx_site_config/tasks/main.yml b/ansible/roles/nginx_site_config/tasks/main.yml new file mode 100644 index 0000000..57bcd8e --- /dev/null +++ b/ansible/roles/nginx_site_config/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Add site config file + template: + src: tildes.conf.jinja2 + dest: /etc/nginx/sites-available/tildes.conf + owner: root + group: root + mode: 0644 + notify: + - Reload nginx + +- name: Enable site in nginx + file: + path: /etc/nginx/sites-enabled/tildes.conf + src: /etc/nginx/sites-available/tildes.conf + state: link + owner: root + group: root + mode: 0644 + notify: + - Reload nginx diff --git a/salt/salt/nginx/tildes.conf.jinja2 b/ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 similarity index 87% rename from salt/salt/nginx/tildes.conf.jinja2 rename to ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 index df7749e..92795cf 100644 --- a/salt/salt/nginx/tildes.conf.jinja2 +++ b/ansible/roles/nginx_site_config/templates/tildes.conf.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir -%} upstream app_server { server unix:/run/gunicorn/socket fail_timeout=0; } @@ -35,9 +34,11 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; - {% if grains['id'] != 'dev' %} - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; + {% if nginx_enable_hsts %} + add_header Strict-Transport-Security "max-age={{ hsts_max_age }}; includeSubDomains; preload" always; + {% endif %} + {% if nginx_enable_csp %} # Content Security Policy: # - "img-src data:" is needed for Spectre.css icons # - "https://js.stripe.com" in script-src and frame-src is needed for Stripe @@ -49,7 +50,7 @@ server { add_header X-Xss-Protection "1; mode=block" always; add_header Referrer-Policy "strict-origin-when-cross-origin" always; - server_name {{ pillar['site_hostname'] }}; + server_name {{ site_hostname }}; keepalive_timeout 5; @@ -57,9 +58,9 @@ server { # Block access to /metrics except from Prometheus server(s) location /metrics { - {% for ip in pillar['prometheus_ips'] %} + {% for ip in prometheus_ips -%} allow {{ ip }}; - {% endfor %} + {% endfor -%} deny all; # try_files is unnecessary here, but I don't know the "proper" way @@ -76,7 +77,7 @@ server { } location @proxy_to_app { - {% if grains["id"] == "prod" %} + {% if nginx_enable_ratelimiting %} # apply rate-limiting, allowing a burst above the limit limit_req zone=tildes_app burst=5 nodelay; {% endif -%} @@ -96,14 +97,14 @@ server { } } -{% if grains["id"] == "prod" %} +{% if nginx_redirect_www %} # redirect www. to base domain server { listen 80; listen 443 ssl http2; listen [::]:443 ssl http2; - server_name www.tildes.net; - return 301 https://tildes.net$request_uri; + server_name www.{{ site_hostname }}; + return 301 https://{{ site_hostname }}$request_uri; } {% endif %} diff --git a/ansible/roles/nodejs/tasks/main.yml b/ansible/roles/nodejs/tasks/main.yml new file mode 100644 index 0000000..2138af3 --- /dev/null +++ b/ansible/roles/nodejs/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Add APT key for NodeSource Node.js repository + apt_key: + url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key + +- name: Add NodeSource Node.js APT repository + apt_repository: + repo: deb https://deb.nodesource.com/node_14.x buster main + +- name: Install Node.js + apt: + name: nodejs + +- name: Install npm packages defined in package.json + become_user: "{{ app_username }}" + community.general.npm: + path: "{{ app_dir }}" + # --no-bin-links option is needed to prevent npm from creating symlinks in the .bin + # directory, which doesn't work inside Vagrant on Windows + no_bin_links: true diff --git a/ansible/roles/pgbouncer/handlers/main.yml b/ansible/roles/pgbouncer/handlers/main.yml new file mode 100644 index 0000000..0d14811 --- /dev/null +++ b/ansible/roles/pgbouncer/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload pgbouncer + service: + name: pgbouncer + state: reloaded diff --git a/ansible/roles/pgbouncer/meta/main.yml b/ansible/roles/pgbouncer/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/pgbouncer/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/pgbouncer/tasks/main.yml b/ansible/roles/pgbouncer/tasks/main.yml new file mode 100644 index 0000000..ce187b6 --- /dev/null +++ b/ansible/roles/pgbouncer/tasks/main.yml @@ -0,0 +1,31 @@ +--- +- name: Install pgbouncer + apt: + name: pgbouncer + +- name: Add pgbouncer.ini + template: + src: pgbouncer.ini.jinja2 + dest: /etc/pgbouncer/pgbouncer.ini + owner: postgres + group: postgres + mode: 0640 + notify: + - Reload pgbouncer + +- name: Add user to pgbouncer userlist + lineinfile: + path: /etc/pgbouncer/userlist.txt + line: '"tildes" ""' + create: true + owner: postgres + group: postgres + mode: 0640 + notify: + - Reload pgbouncer + +- name: Start and enable pgbouncer service + service: + name: pgbouncer + state: started + enabled: true diff --git a/salt/salt/postgresql/pgbouncer.ini.jinja2 b/ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 similarity index 64% rename from salt/salt/postgresql/pgbouncer.ini.jinja2 rename to ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 index 9b89c8b..da7b5fe 100644 --- a/salt/salt/postgresql/pgbouncer.ini.jinja2 +++ b/ansible/roles/pgbouncer/templates/pgbouncer.ini.jinja2 @@ -1,7 +1,4 @@ [databases] -{% for line in accumulator['pgbouncer_db_lines'] -%} -{{ line }} -{% endfor %} [pgbouncer] logfile = /var/log/postgresql/pgbouncer.log @@ -16,6 +13,6 @@ unix_socket_dir = /var/run/postgresql auth_type = hba auth_file = /etc/pgbouncer/userlist.txt -auth_hba_file = /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf +auth_hba_file = /etc/postgresql/{{ postgresql_version }}/main/pg_hba.conf pool_mode = transaction diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml new file mode 100644 index 0000000..6420521 --- /dev/null +++ b/ansible/roles/postgresql/defaults/main.yml @@ -0,0 +1,12 @@ +--- +postgresql_version: 12 + +# Users of this role can define postgresql_settings, which will be merged with +# this base _postgresql_settings +_postgresql_settings: + lock_timeout: 5000 + statement_timeout: 5000 + idle_in_transaction_session_timeout: 600000 + timezone: "'UTC'" + shared_preload_libraries: "'pg_stat_statements'" +postgresql_settings: {} diff --git a/ansible/roles/postgresql/handlers/main.yml b/ansible/roles/postgresql/handlers/main.yml new file mode 100644 index 0000000..4a6d976 --- /dev/null +++ b/ansible/roles/postgresql/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: Restart postgresql + service: + name: postgresql + state: restarted + +- name: Reload postgresql + service: + name: postgresql + state: reloaded diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml new file mode 100644 index 0000000..f925e67 --- /dev/null +++ b/ansible/roles/postgresql/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Add APT key for PostgreSQL repository + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + +- name: Add PostgreSQL APT repository + apt_repository: + repo: deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main + +- name: Install PostgreSQL + apt: + name: postgresql-{{ postgresql_version }} + +- name: Start and enable PostgreSQL service + service: + name: postgresql + state: started + enabled: true + +- name: Set configuration options in postgresql.conf + lineinfile: + path: /etc/postgresql/{{ postgresql_version }}/main/postgresql.conf + regexp: "^#?{{ item.key }} ?=" + line: "{{ item.key }} = {{ item.value }}" + loop: "{{ _postgresql_settings | combine(postgresql_settings) | dict2items }}" + notify: + - Restart postgresql diff --git a/ansible/roles/postgresql_plpython/meta/main.yml b/ansible/roles/postgresql_plpython/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/postgresql_plpython/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/postgresql_plpython/tasks/main.yml b/ansible/roles/postgresql_plpython/tasks/main.yml new file mode 100644 index 0000000..c6fab8d --- /dev/null +++ b/ansible/roles/postgresql_plpython/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Install PL/Python3 procedural language for PostgreSQL + apt: + name: postgresql-plpython3-{{ postgresql_version }} + +- name: Set PYTHONPATH env var for PostgreSQL + lineinfile: + path: /etc/postgresql/{{ postgresql_version }}/main/environment + regexp: "^PYTHONPATH=" + line: "PYTHONPATH='{{ venv_dir }}/lib/python{{ python_version }}/site-packages:{{ app_dir }}'" + notify: + - Restart postgresql diff --git a/ansible/roles/postgresql_redis_bridge/meta/main.yml b/ansible/roles/postgresql_redis_bridge/meta/main.yml new file mode 100644 index 0000000..edeef00 --- /dev/null +++ b/ansible/roles/postgresql_redis_bridge/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: postgresql + - role: redis diff --git a/ansible/roles/postgresql_redis_bridge/tasks/main.yml b/ansible/roles/postgresql_redis_bridge/tasks/main.yml new file mode 100644 index 0000000..d517275 --- /dev/null +++ b/ansible/roles/postgresql_redis_bridge/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Create postgresql_redis_bridge service file + template: + src: postgresql_redis_bridge.service.jinja2 + dest: /etc/systemd/system/postgresql_redis_bridge.service + owner: root + group: root + mode: 0644 + +- name: Start and enable postgresql_redis_bridge service + service: + name: postgresql_redis_bridge + state: started + enabled: true diff --git a/salt/salt/postgresql_redis_bridge.service.jinja2 b/ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 similarity index 73% rename from salt/salt/postgresql_redis_bridge.service.jinja2 rename to ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 index be2a5f3..df485cb 100644 --- a/salt/salt/postgresql_redis_bridge.service.jinja2 +++ b/ansible/roles/postgresql_redis_bridge/templates/postgresql_redis_bridge.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=postgresql_redis_bridge - convert NOTIFY to Redis streams Requires=redis.service @@ -9,7 +8,7 @@ PartOf=redis.service User={{ app_username }} Group={{ app_username }} WorkingDirectory={{ app_dir }}/scripts -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" +Environment="INI_FILE={{ app_dir }}/{{ ini_file }}" ExecStart={{ bin_dir }}/python postgresql_redis_bridge.py Restart=always RestartSec=5 diff --git a/ansible/roles/postgresql_tildes_dbs/defaults/main.yml b/ansible/roles/postgresql_tildes_dbs/defaults/main.yml new file mode 100644 index 0000000..52acffa --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/defaults/main.yml @@ -0,0 +1,6 @@ +--- +postgresql_tildes_databases: + - tildes +postgresql_tildes_user_flags: "" + +tildes_database_insert_dev_data: false diff --git a/ansible/roles/postgresql_tildes_dbs/meta/main.yml b/ansible/roles/postgresql_tildes_dbs/meta/main.yml new file mode 100644 index 0000000..008e938 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: postgresql + - role: pgbouncer + + # needed to be able to run the db init scripts + - role: python + - role: cmark-gfm diff --git a/ansible/roles/postgresql_tildes_dbs/tasks/database.yml b/ansible/roles/postgresql_tildes_dbs/tasks/database.yml new file mode 100644 index 0000000..4be87f5 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/tasks/database.yml @@ -0,0 +1,38 @@ +--- +- name: Create database and enable access and all necessary extensions + become_user: postgres + block: + - name: Create database + community.postgresql.postgresql_db: + name: "{{ item }}" + owner: tildes + + - name: Enable extensions + community.postgresql.postgresql_ext: + name: "{{ extension }}" + db: "{{ item }}" + loop: "{{ extensions }}" + # since the "outer" loop by include_tasks is already using the `item` variable, + # this will use `extension` for the loop inside here + loop_control: + loop_var: extension + + - name: Add database to pg_hba.conf + community.postgresql.postgresql_pg_hba: + dest: /etc/postgresql/{{ postgresql_version }}/main/pg_hba.conf + databases: "{{ item }}" + users: tildes + contype: local + method: trust + notify: + - Reload postgresql + + - name: Add database to pgbouncer.ini + become_user: root + lineinfile: + path: /etc/pgbouncer/pgbouncer.ini + line: "{{ item }} =" + insertafter: "^\\[databases\\]$" + firstmatch: true + notify: + - Reload pgbouncer diff --git a/ansible/roles/postgresql_tildes_dbs/tasks/main.yml b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml new file mode 100644 index 0000000..3911715 --- /dev/null +++ b/ansible/roles/postgresql_tildes_dbs/tasks/main.yml @@ -0,0 +1,62 @@ +--- +- name: Install requirements for building psycopg2 (needed by Ansible) + apt: + name: + - gcc + - libpq-dev + - python3-dev + +- name: Install packages needed by Ansible community plugins + pip: + executable: pip3 + name: + - ipaddress + - psycopg2 + +- name: Create tildes user + become_user: postgres + community.postgresql.postgresql_user: + name: tildes + role_attr_flags: "{{ postgresql_tildes_user_flags }}" + +# This is a bit of a hack to effectively enable looping over a block of tasks +- name: Set up site database (and test database in dev version) + include_tasks: database.yml + loop: "{{ postgresql_tildes_databases }}" + vars: + extensions: + - citext + - ltree + - intarray + - pg_stat_statements + - pg_trgm + - plpython3u + register: database_changes + +# Since handlers don't run until the end of the entire playbook, we need to run them +# manually at this point in case postgresql or pgbouncer need to be reloaded +- name: Trigger handlers to run manually for postgresql/pgbouncer updates + meta: flush_handlers + +- name: Check if the database has already been initialized (will fail if not) + become_user: postgres + community.postgresql.postgresql_query: + db: tildes + query: select user_id from users; + ignore_errors: true + register: users_query + +- name: Initialize the database + become_user: postgres + command: + cmd: "{{ bin_dir }}/python -c \"from scripts.initialize_db import initialize_db; initialize_db('{{ app_dir }}/{{ ini_file }}')\"" + chdir: "{{ app_dir }}" + when: users_query is failed + register: initialize_db + +- name: Insert dev data into database + become_user: "{{ app_username }}" + command: + cmd: "{{ bin_dir }}/python -c \"from scripts.initialize_db import insert_dev_data; insert_dev_data('{{ app_dir }}/{{ ini_file }}')\"" + chdir: "{{ app_dir }}" + when: tildes_database_insert_dev_data and initialize_db is changed diff --git a/ansible/roles/prometheus/defaults/main.yml b/ansible/roles/prometheus/defaults/main.yml new file mode 100644 index 0000000..0a963aa --- /dev/null +++ b/ansible/roles/prometheus/defaults/main.yml @@ -0,0 +1,8 @@ +--- +prometheus_version: 2.0.0 + +prometheus_consumer_scrape_targets: + comment_user_mentions_generator: 25010 + post_processing_script_runner: 25016 + topic_interesting_activity_updater: 25013 + topic_metadata_generator: 25014 diff --git a/salt/salt/prometheus/prometheus.service b/ansible/roles/prometheus/files/prometheus.service similarity index 100% rename from salt/salt/prometheus/prometheus.service rename to ansible/roles/prometheus/files/prometheus.service diff --git a/ansible/roles/prometheus/handlers/main.yml b/ansible/roles/prometheus/handlers/main.yml new file mode 100644 index 0000000..2f5a4b8 --- /dev/null +++ b/ansible/roles/prometheus/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus + service: + name: prometheus + state: restarted diff --git a/ansible/roles/prometheus/tasks/main.yml b/ansible/roles/prometheus/tasks/main.yml new file mode 100644 index 0000000..292156d --- /dev/null +++ b/ansible/roles/prometheus/tasks/main.yml @@ -0,0 +1,59 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Check if Prometheus is installed + stat: + path: /opt/prometheus/prometheus + register: prometheus_binary + +- name: Download and install Prometheus + when: not prometheus_binary.stat.exists + block: + - name: Download Prometheus from GitHub + get_url: + dest: /tmp/prometheus-{{ prometheus_version }}.tar.gz + url: https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz + checksum: sha256:e12917b25b32980daee0e9cf879d9ec197e2893924bd1574604eb0f550034d46 + + - name: Create Prometheus directory + file: + path: /opt/prometheus + state: directory + owner: prometheus + group: prometheus + mode: 0755 + + - name: Extract Prometheus + unarchive: + remote_src: true + src: /tmp/prometheus-{{ prometheus_version }}.tar.gz + dest: /opt/prometheus + owner: prometheus + group: prometheus + extra_opts: + - --strip-components=1 + +- name: Create Prometheus service file + copy: + src: prometheus.service + dest: /etc/systemd/system/prometheus.service + owner: root + group: root + mode: 0644 + +- name: Add Prometheus config file + template: + src: prometheus.yml.jinja2 + dest: /opt/prometheus/prometheus.yml + owner: prometheus + group: prometheus + mode: 0644 + notify: + - Restart prometheus + +- name: Start and enable prometheus service + service: + name: prometheus + state: started + enabled: true diff --git a/salt/salt/prometheus/prometheus.yml.jinja2 b/ansible/roles/prometheus/templates/prometheus.yml.jinja2 similarity index 53% rename from salt/salt/prometheus/prometheus.yml.jinja2 rename to ansible/roles/prometheus/templates/prometheus.yml.jinja2 index 1907a31..4fea515 100644 --- a/salt/salt/prometheus/prometheus.yml.jinja2 +++ b/ansible/roles/prometheus/templates/prometheus.yml.jinja2 @@ -5,21 +5,21 @@ global: scrape_configs: - job_name: "node" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9100'] + - targets: ['{{ site_hostname }}:9100'] - job_name: "redis" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9121'] + - targets: ['{{ site_hostname }}:9121'] - job_name: "postgres" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:9187'] + - targets: ['{{ site_hostname }}:9187'] - job_name: "tildes" scheme: https static_configs: - - targets: ['{{ pillar['site_hostname'] }}:443'] - {% if grains['id'] == 'dev' %} + - targets: ['{{ site_hostname }}:443'] + {% if site_hostname == "localhost" -%} tls_config: insecure_skip_verify: true {% endif %} @@ -31,7 +31,7 @@ scrape_configs: module: [site_ipv4] static_configs: - targets: - - https://{{ pillar['site_hostname'] }} + - https://{{ site_hostname }} relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -47,7 +47,7 @@ scrape_configs: module: [site_ipv6] static_configs: - targets: - - https://{{ pillar['site_hostname'] }} + - https://{{ site_hostname }} relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -57,28 +57,8 @@ scrape_configs: replacement: 127.0.0.1:9115 # The blackbox exporter's real hostname:port # event stream consumers (background jobs) - - job_name: "consumer_comment_user_mentions_generator" + {% for name, port in prometheus_consumer_scrape_targets.items() -%} + - job_name: "consumer_{{ name }}" static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25010'] - - - job_name: "consumer_topic_interesting_activity_updater" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25013'] - - - job_name: "consumer_topic_metadata_generator" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25014'] - - {% if grains["id"] != "dev" %} - - job_name: "consumer_site_icon_downloader" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25011'] - - - job_name: "consumer_topic_embedly_extractor" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25012'] - - - job_name: "consumer_topic_youtube_scraper" - static_configs: - - targets: ['{{ pillar['site_hostname'] }}:25015'] - {% endif %} + - targets: ['{{ site_hostname }}:{{ port }}'] + {% endfor %} diff --git a/salt/salt/prometheus/exporters/prometheus_node_exporter.service b/ansible/roles/prometheus_node_exporter/files/prometheus_node_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_node_exporter.service rename to ansible/roles/prometheus_node_exporter/files/prometheus_node_exporter.service diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml new file mode 100644 index 0000000..e7cac53 --- /dev/null +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Download node_exporter from GitHub + get_url: + dest: /tmp/prometheus_node_exporter.tar.gz + url: https://github.com/prometheus/node_exporter/releases/download/v0.13.0/node_exporter-0.13.0.linux-amd64.tar.gz + checksum: sha256:2de5d1e51330c41588ed4c88bc531a3d2dccf6b4d7b99d5782d95cff27a3c049 + +- name: Create node_exporter directory + file: + path: /opt/prometheus_node_exporter + state: directory + owner: prometheus + group: prometheus + mode: 0755 + +- name: Extract node_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_node_exporter.tar.gz + dest: /opt/prometheus_node_exporter + owner: prometheus + group: prometheus + extra_opts: + - --strip-components=1 + creates: /opt/prometheus_node_exporter/node_exporter + +- name: Create node_exporter service file + copy: + src: prometheus_node_exporter.service + dest: /etc/systemd/system/prometheus_node_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable node_exporter service + service: + name: prometheus_node_exporter + state: started + enabled: true diff --git a/salt/salt/prometheus/exporters/prometheus_postgres_exporter.service b/ansible/roles/prometheus_postgres_exporter/files/prometheus_postgres_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_postgres_exporter.service rename to ansible/roles/prometheus_postgres_exporter/files/prometheus_postgres_exporter.service diff --git a/salt/salt/prometheus/exporters/queries.yaml b/ansible/roles/prometheus_postgres_exporter/files/queries.yaml similarity index 100% rename from salt/salt/prometheus/exporters/queries.yaml rename to ansible/roles/prometheus_postgres_exporter/files/queries.yaml diff --git a/ansible/roles/prometheus_postgres_exporter/meta/main.yml b/ansible/roles/prometheus_postgres_exporter/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/prometheus_postgres_exporter/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/prometheus_postgres_exporter/tasks/main.yml b/ansible/roles/prometheus_postgres_exporter/tasks/main.yml new file mode 100644 index 0000000..55cd99d --- /dev/null +++ b/ansible/roles/prometheus_postgres_exporter/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Download postgres_exporter from GitHub + get_url: + dest: /tmp/prometheus_postgres_exporter.tar.gz + url: https://github.com/wrouesnel/postgres_exporter/releases/download/v0.4.7/postgres_exporter_v0.4.7_linux-amd64.tar.gz + checksum: sha256:c34d61bb4deba8efae06fd3c9979b96dae3f3c757698ce3384c80fff586c667b + +- name: Create postgres_exporter directory + file: + path: /opt/prometheus_postgres_exporter + state: directory + owner: postgres + group: postgres + mode: 0755 + +- name: Extract postgres_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_postgres_exporter.tar.gz + dest: /opt/prometheus_postgres_exporter + owner: postgres + group: postgres + extra_opts: + - --strip-components=1 + creates: /opt/prometheus_postgres_exporter/postgres_exporter + +- name: Create queries.yaml file + copy: + src: queries.yaml + dest: /opt/prometheus_postgres_exporter/queries.yaml + owner: postgres + group: postgres + mode: 0644 + +- name: Create postgres_exporter service file + copy: + src: prometheus_postgres_exporter.service + dest: /etc/systemd/system/prometheus_postgres_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable postgres_exporter service + service: + name: prometheus_postgres_exporter + state: started + enabled: true diff --git a/salt/salt/prometheus/exporters/prometheus_redis_exporter.service b/ansible/roles/prometheus_redis_exporter/files/prometheus_redis_exporter.service similarity index 100% rename from salt/salt/prometheus/exporters/prometheus_redis_exporter.service rename to ansible/roles/prometheus_redis_exporter/files/prometheus_redis_exporter.service diff --git a/ansible/roles/prometheus_redis_exporter/meta/main.yml b/ansible/roles/prometheus_redis_exporter/meta/main.yml new file mode 100644 index 0000000..89a7c92 --- /dev/null +++ b/ansible/roles/prometheus_redis_exporter/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: redis diff --git a/ansible/roles/prometheus_redis_exporter/tasks/main.yml b/ansible/roles/prometheus_redis_exporter/tasks/main.yml new file mode 100644 index 0000000..bf98d6f --- /dev/null +++ b/ansible/roles/prometheus_redis_exporter/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Create prometheus user and group + import_tasks: prometheus_user.yml + +- name: Download redis_exporter from GitHub + get_url: + dest: /tmp/prometheus_redis_exporter.tar.gz + url: https://github.com/oliver006/redis_exporter/releases/download/v0.26.0/redis_exporter-v0.26.0.linux-amd64.tar.gz + checksum: sha256:39354c57b9d02b455c584baf46a2df6ed3d1ac190c88e3ec0fa0c23b49ccc2bb + +- name: Create redis_exporter directory + file: + path: /opt/prometheus_redis_exporter + state: directory + owner: prometheus + group: prometheus + mode: 0755 + +- name: Extract redis_exporter + unarchive: + remote_src: true + src: /tmp/prometheus_redis_exporter.tar.gz + dest: /opt/prometheus_redis_exporter + owner: prometheus + group: prometheus + creates: /opt/prometheus_redis_exporter/redis_exporter + +- name: Create redis_exporter service file + copy: + src: prometheus_redis_exporter.service + dest: /etc/systemd/system/prometheus_redis_exporter.service + owner: root + group: root + mode: 0644 + +- name: Start and enable redis_exporter service + service: + name: prometheus_redis_exporter + state: started + enabled: true diff --git a/ansible/roles/pts_lbsearch/tasks/main.yml b/ansible/roles/pts_lbsearch/tasks/main.yml new file mode 100644 index 0000000..703be93 --- /dev/null +++ b/ansible/roles/pts_lbsearch/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Download pts_lbsearch code from GitHub + get_url: + dest: /tmp/pts_lbsearch.c + url: https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c + checksum: sha256:ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1 + +- name: Compile pts_lbsearch + command: + chdir: /tmp + # compilation command taken from the top of the source file + cmd: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c + creates: /usr/local/bin/pts_lbsearch diff --git a/ansible/roles/python/defaults/main.yml b/ansible/roles/python/defaults/main.yml new file mode 100644 index 0000000..420c1cc --- /dev/null +++ b/ansible/roles/python/defaults/main.yml @@ -0,0 +1,2 @@ +--- +pip_requirements_filename: requirements.txt diff --git a/ansible/roles/python/tasks/main.yml b/ansible/roles/python/tasks/main.yml new file mode 100644 index 0000000..4bf9c62 --- /dev/null +++ b/ansible/roles/python/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Check if the correct version of Python is already installed + stat: + path: /usr/local/bin/python{{ python_version }} + register: python_binary + +- name: Download and install Python + when: not python_binary.stat.exists + block: + - name: Download Python source code + get_url: + dest: /tmp/python.tar.gz + url: https://www.python.org/ftp/python/3.8.10/Python-3.8.10.tgz + checksum: sha256:b37ac74d2cbad2590e7cd0dd2b3826c29afe89a734090a87bf8c03c45066cb65 + + - name: Create temp directory to extract Python to + file: + path: /tmp/python + state: directory + + - name: Extract Python + unarchive: + remote_src: true + src: /tmp/python.tar.gz + dest: /tmp/python + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for Python + apt: + name: + - make + - build-essential + - libssl-dev + - zlib1g-dev + - libbz2-dev + - libreadline-dev + - libsqlite3-dev + - wget + - curl + - llvm + - libncurses5-dev + - libncursesw5-dev + - xz-utils + - tk-dev + + - name: Build and install Python (this can take a long time) + shell: + chdir: /tmp/python + cmd: | + ./configure --enable-optimizations --with-ensurepip=install + make + make altinstall + +- name: Create dir for venvs + file: + path: /opt/venvs + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0755 + +- name: Install packages needed for compiling psycopg2 + apt: + name: + - gcc + - libpq-dev + +- name: Create venv + become_user: "{{ app_username }}" + command: + cmd: python{{ python_version }} -m venv {{ venv_dir }} + creates: "{{ venv_dir }}" + +- name: Check installed packages in venv + command: + cmd: "{{ bin_dir }}/pip freeze" + register: pip_freeze + changed_when: false + +- name: Install Python packages into venv + become_user: "{{ app_username }}" + pip: + executable: "{{ bin_dir }}/pip" + requirements: "{{ app_dir }}/{{ pip_requirements_filename }}" + +- name: Install Tildes into venv as editable + become_user: "{{ app_username }}" + pip: + executable: "{{ bin_dir }}/pip" + name: "{{ app_dir }}" + editable: true + when: "'-e '+app_dir not in pip_freeze.stdout_lines" diff --git a/ansible/roles/redis/defaults/main.yml b/ansible/roles/redis/defaults/main.yml new file mode 100644 index 0000000..659e9a2 --- /dev/null +++ b/ansible/roles/redis/defaults/main.yml @@ -0,0 +1,2 @@ +--- +redis_version: 5.0.7 diff --git a/salt/salt/redis/redis.service b/ansible/roles/redis/files/redis.service similarity index 100% rename from salt/salt/redis/redis.service rename to ansible/roles/redis/files/redis.service diff --git a/salt/salt/redis/transparent_hugepage.service b/ansible/roles/redis/files/transparent_hugepage.service similarity index 100% rename from salt/salt/redis/transparent_hugepage.service rename to ansible/roles/redis/files/transparent_hugepage.service diff --git a/ansible/roles/redis/handlers/main.yml b/ansible/roles/redis/handlers/main.yml new file mode 100644 index 0000000..1ba7b3d --- /dev/null +++ b/ansible/roles/redis/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart redis + service: + name: redis + state: restarted diff --git a/ansible/roles/redis/tasks/main.yml b/ansible/roles/redis/tasks/main.yml new file mode 100644 index 0000000..9a76822 --- /dev/null +++ b/ansible/roles/redis/tasks/main.yml @@ -0,0 +1,131 @@ +--- +- name: Check if Redis is installed + stat: + path: /usr/local/bin/redis-server + register: redis_server + +- name: Download and install Redis + when: not redis_server.stat.exists + block: + - name: Download Redis from GitHub + get_url: + dest: /tmp/redis-{{ redis_version }}.tar.gz + url: https://github.com/antirez/redis/archive/{{ redis_version }}.tar.gz + checksum: sha256:2761422599f8969559e66797cd7f606c16e907bf82d962345a7d366c5d1278df + + - name: Create temp directory to extract Redis to + file: + path: /tmp/redis-{{ redis_version }} + state: directory + + - name: Extract Redis + unarchive: + remote_src: true + src: /tmp/redis-{{ redis_version }}.tar.gz + dest: /tmp/redis-{{ redis_version }} + extra_opts: + - --strip-components=1 + + - name: Install build dependencies for Redis + apt: + name: build-essential + + - name: Install Redis + shell: + chdir: /tmp/redis-{{ redis_version }} + cmd: | + make + make install + +- name: Create group for redis user + group: + name: redis + +- name: Create redis user + user: + name: redis + group: redis + create_home: false + +- name: Create /run/redis + file: + path: /run/redis + state: directory + owner: redis + group: redis + mode: 0755 + +- name: Create /var/lib/redis + file: + path: /var/lib/redis + state: directory + owner: redis + group: redis + mode: 0700 + +- name: Create /var/log/redis + file: + path: /var/log/redis + state: directory + owner: redis + group: redis + mode: 0744 + +- name: Create Redis configuration file + template: + src: redis.conf.jinja2 + dest: /etc/redis.conf + owner: redis + group: redis + mode: 0600 + +- name: Create Redis service file + copy: + src: redis.service + dest: /etc/systemd/system/redis.service + owner: root + group: root + mode: 0644 + +- name: Add service file for disabling transparent hugepage + copy: + src: transparent_hugepage.service + dest: /etc/systemd/system/transparent_hugepage.service + owner: root + group: root + mode: 0644 + +- name: Check if transparent hugepage is currently enabled + command: + cmd: cat /sys/kernel/mm/transparent_hugepage/enabled + register: transparent_hugepage + changed_when: false + +- name: Start and enable "disable transparent hugepage" service + service: + name: transparent_hugepage.service + state: started + enabled: true + when: "'[never]' not in transparent_hugepage.stdout" + +- name: Check if kernel overcommit mode is already set + command: + cmd: sysctl -n vm.overcommit_memory + register: overcommit_memory + changed_when: false + +- name: Set kernel overcommit mode temporarily (recommended by Redis) + command: + cmd: sysctl vm.overcommit_memory=1 + when: overcommit_memory.stdout == "0" + +- name: Make kernel overcommit mode permanent (recommended by Redis, requires restart) + lineinfile: + path: /etc/sysctl.conf + line: vm.overcommit_memory = 1 + +- name: Start and enable redis service + service: + name: redis + state: started + enabled: true diff --git a/salt/salt/redis/redis.conf.jinja2 b/ansible/roles/redis/templates/redis.conf.jinja2 similarity index 99% rename from salt/salt/redis/redis.conf.jinja2 rename to ansible/roles/redis/templates/redis.conf.jinja2 index d032c42..0c597d3 100644 --- a/salt/salt/redis/redis.conf.jinja2 +++ b/ansible/roles/redis/templates/redis.conf.jinja2 @@ -43,12 +43,6 @@ # loadmodule /path/to/my_module.so # loadmodule /path/to/other_module.so -{% if accumulator is defined %} -{% for line in accumulator['redis_loadmodule_lines'] -%} -{{ line }} -{% endfor %} -{% endif %} - ################################## NETWORK ##################################### # By default, if no "bind" configuration directive is specified, Redis listens diff --git a/ansible/roles/redis_module_cell/meta/main.yml b/ansible/roles/redis_module_cell/meta/main.yml new file mode 100644 index 0000000..89a7c92 --- /dev/null +++ b/ansible/roles/redis_module_cell/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: redis diff --git a/ansible/roles/redis_module_cell/tasks/main.yml b/ansible/roles/redis_module_cell/tasks/main.yml new file mode 100644 index 0000000..9ef2d62 --- /dev/null +++ b/ansible/roles/redis_module_cell/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Download redis-cell Redis module from GitHub + get_url: + dest: /tmp/redis-cell.tar.gz + url: https://github.com/brandur/redis-cell/releases/download/v0.2.1/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz + checksum: sha256:9427fb100f4cada817f30f854ead7f233de32948a0ec644f15988c275a2ed1cb + +- name: Create /opt/redis-cell + file: + path: /opt/redis-cell + state: directory + owner: redis + group: redis + mode: 0755 + +- name: Extract redis-cell + unarchive: + remote_src: true + src: /tmp/redis-cell.tar.gz + dest: /opt/redis-cell + +- name: Load redis-cell module in Redis configuration + lineinfile: + path: /etc/redis.conf + line: loadmodule /opt/redis-cell/libredis_cell.so + notify: + - Restart redis diff --git a/ansible/roles/scripts/tasks/main.yml b/ansible/roles/scripts/tasks/main.yml new file mode 100644 index 0000000..cb9890d --- /dev/null +++ b/ansible/roles/scripts/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Install the activate script + template: + src: activate.sh.jinja2 + dest: /usr/local/bin/activate + owner: root + group: root + mode: 0755 diff --git a/salt/salt/scripts/activate.sh.jinja2 b/ansible/roles/scripts/templates/activate.sh.jinja2 similarity index 78% rename from salt/salt/scripts/activate.sh.jinja2 rename to ansible/roles/scripts/templates/activate.sh.jinja2 index 014536c..21871df 100644 --- a/salt/salt/scripts/activate.sh.jinja2 +++ b/ansible/roles/scripts/templates/activate.sh.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} #!/bin/bash # # Simple convenience script to activate the venv and switch to the app dir diff --git a/ansible/roles/self_signed_ssl_cert/meta/main.yml b/ansible/roles/self_signed_ssl_cert/meta/main.yml new file mode 100644 index 0000000..8b662c9 --- /dev/null +++ b/ansible/roles/self_signed_ssl_cert/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: nginx diff --git a/ansible/roles/self_signed_ssl_cert/tasks/main.yml b/ansible/roles/self_signed_ssl_cert/tasks/main.yml new file mode 100644 index 0000000..e9bd31c --- /dev/null +++ b/ansible/roles/self_signed_ssl_cert/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Install packages needed by Ansible community plugins + pip: + executable: pip3 + name: cryptography + +- name: Create directory for certificate + file: + path: "{{ ssl_cert_dir }}" + state: directory + mode: 0755 + +- name: Create a private key + community.crypto.openssl_privatekey: + path: "{{ ssl_private_key_path }}" + +- name: Create a self-signed certificate + community.crypto.x509_certificate: + path: "{{ ssl_cert_path }}" + privatekey_path: "{{ ssl_private_key_path }}" + provider: selfsigned + notify: + - Reload nginx diff --git a/ansible/roles/webassets/meta/main.yml b/ansible/roles/webassets/meta/main.yml new file mode 100644 index 0000000..f7230ad --- /dev/null +++ b/ansible/roles/webassets/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: python diff --git a/ansible/roles/webassets/tasks/main.yml b/ansible/roles/webassets/tasks/main.yml new file mode 100644 index 0000000..0ce8619 --- /dev/null +++ b/ansible/roles/webassets/tasks/main.yml @@ -0,0 +1,29 @@ +--- +- name: Check if site-icons.css file exists + stat: + path: "{{ app_dir }}/static/css/site-icons.css" + register: site_icons_css_file + +# webassets will crash the site if this file doesn't exist +- name: Create site-icons.css file + file: + path: "{{ app_dir }}/static/css/site-icons.css" + state: touch + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0644 + when: not site_icons_css_file.stat.exists + +- name: Create systemd service file + template: + src: webassets.service.jinja2 + dest: /etc/systemd/system/webassets.service + owner: root + group: root + mode: 0644 + +- name: Start and enable webassets service + service: + name: webassets + state: started + enabled: true diff --git a/salt/salt/webassets.service.jinja2 b/ansible/roles/webassets/templates/webassets.service.jinja2 similarity index 80% rename from salt/salt/webassets.service.jinja2 rename to ansible/roles/webassets/templates/webassets.service.jinja2 index 1c3ad7d..f84355a 100644 --- a/salt/salt/webassets.service.jinja2 +++ b/ansible/roles/webassets/templates/webassets.service.jinja2 @@ -1,4 +1,3 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Webassets - auto-compile JS files on change diff --git a/ansible/roles/wiki_repo/tasks/main.yml b/ansible/roles/wiki_repo/tasks/main.yml new file mode 100644 index 0000000..e5ed427 --- /dev/null +++ b/ansible/roles/wiki_repo/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: Create the base repo directory + file: + path: /var/lib/tildes-wiki + state: directory + owner: "{{ app_username }}" + group: "{{ app_username }}" + mode: 0775 + +- name: Check if a wiki git repo exists + stat: + path: /var/lib/tildes-wiki/.git + register: wiki_repo + +- name: Create a git repo and initial commit + become_user: "{{ app_username }}" + shell: + cmd: | + git init + git config user.name "Tildes" + git config user.email "Tildes" + git commit --allow-empty -m "Initial commit" + chdir: /var/lib/tildes-wiki + when: not wiki_repo.stat.exists diff --git a/ansible/tasks/prometheus_user.yml b/ansible/tasks/prometheus_user.yml new file mode 100644 index 0000000..36a3c19 --- /dev/null +++ b/ansible/tasks/prometheus_user.yml @@ -0,0 +1,10 @@ +--- +- name: Create group for prometheus user + group: + name: prometheus + +- name: Create prometheus user + user: + name: prometheus + group: prometheus + create_home: false diff --git a/ansible/vars.yml b/ansible/vars.yml new file mode 100644 index 0000000..4012c89 --- /dev/null +++ b/ansible/vars.yml @@ -0,0 +1,8 @@ +--- +app_dir: /opt/tildes +venv_dir: /opt/venvs/tildes +bin_dir: "{{ venv_dir }}/bin" + +static_sites_dir: /opt/tildes-static-sites + +python_version: 3.8 diff --git a/salt/minion b/salt/minion deleted file mode 100644 index fe4e094..0000000 --- a/salt/minion +++ /dev/null @@ -1,19 +0,0 @@ -# look for files on the minion, not the master -file_client: local - -# set a specific minion ID for use in top file / pillars -# options: -# - "dev" for the vagrant setup -# - "prod" for (single-server) production -# - "monitoring" for the monitoring server -id: dev - -# base path for SSL certificates -ca.cert_base_path: '/etc/pki' - -state_verbose: False - -# enable new module.run state syntax -# https://docs.saltstack.com/en/develop/topics/releases/2017.7.0.html#state-module-changes -use_superseded: - - module.run diff --git a/salt/pillar/dev.sls b/salt/pillar/dev.sls deleted file mode 100644 index 6447f76..0000000 --- a/salt/pillar/dev.sls +++ /dev/null @@ -1,8 +0,0 @@ -gunicorn_args: --reload -ini_file: development.ini -ssl_cert_path: /etc/pki/tls/certs/localhost.crt -ssl_private_key_path: /etc/pki/tls/certs/localhost.key -nginx_worker_processes: 1 -postgresql_version: 12 -prometheus_ips: ['127.0.0.1'] -site_hostname: localhost diff --git a/salt/pillar/monitoring-secrets.example.sls b/salt/pillar/monitoring-secrets.example.sls deleted file mode 100644 index c505701..0000000 --- a/salt/pillar/monitoring-secrets.example.sls +++ /dev/null @@ -1,8 +0,0 @@ -sentry_secret: 'sentry_secret_token' -sentry_email: 'sentry_superuser@email.address' -sentry_password: 'passwordforsentrysuperuser' -sentry_server_name: 'sentry.example.com' -developer_ips: ['127.0.0.1'] -prometheus_server_name: 'prom.example.com' -grafana_server_name: 'grafana.example.com' -grafana_admin_password: 'passwordforgrafanaadmin' diff --git a/salt/pillar/monitoring.sls b/salt/pillar/monitoring.sls deleted file mode 100644 index 004b89a..0000000 --- a/salt/pillar/monitoring.sls +++ /dev/null @@ -1,6 +0,0 @@ -ssl_cert_path: /etc/letsencrypt/live/tildes.net/fullchain.pem -ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem -hsts_max_age: 60 -nginx_worker_processes: auto -postgresql_version: 9.6 -site_hostname: tildes.net diff --git a/salt/pillar/prod.sls b/salt/pillar/prod.sls deleted file mode 100644 index a03a6d6..0000000 --- a/salt/pillar/prod.sls +++ /dev/null @@ -1,11 +0,0 @@ -gunicorn_args: --workers 8 -ini_file: production.ini -ssl_cert_path: /etc/letsencrypt/live/tildes.net/fullchain.pem -ssl_private_key_path: /etc/letsencrypt/live/tildes.net/privkey.pem -hsts_max_age: 63072000 -nginx_worker_processes: auto -postgresql_version: 12 -prometheus_ips: ['2607:5300:201:3100::6e77'] -site_hostname: tildes.net -ipv6_address: '2607:5300:0203:2dd8::' -ipv6_gateway: '2607:5300:0203:2dff:ff:ff:ff:ff' diff --git a/salt/pillar/top.sls b/salt/pillar/top.sls deleted file mode 100644 index 91f58d9..0000000 --- a/salt/pillar/top.sls +++ /dev/null @@ -1,8 +0,0 @@ -base: - 'dev': - - dev - 'prod': - - prod - 'monitoring': - - monitoring - - monitoring-secrets diff --git a/salt/salt/boussole.sls b/salt/salt/boussole.sls deleted file mode 100644 index 9597569..0000000 --- a/salt/salt/boussole.sls +++ /dev/null @@ -1,47 +0,0 @@ -{% from 'common.jinja2' import app_dir, python_version %} - -{% set boussole_venv_dir = '/opt/venvs/boussole' %} - -# Salt seems to use the deprecated pyvenv script, manual for now -boussole-venv-setup: - cmd.run: - - name: python{{ python_version }} -m venv {{ boussole_venv_dir }} - - creates: {{ boussole_venv_dir }} - - require: - - pkg: python{{ python_version }}-venv - -boussole-pip-installs: - cmd.run: - - name: {{ boussole_venv_dir }}/bin/pip install boussole - - unless: ls {{ boussole_venv_dir }}/lib/python{{ python_version }}/site-packages/boussole - -/etc/systemd/system/boussole.service: - file.managed: - - source: salt://boussole.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: boussole.service - -boussole.service: - service.running: - - enable: True - - require: - - cmd: boussole-pip-installs - -create-css-directory: - file.directory: - - name: {{ app_dir }}/static/css - -initial-boussole-run: - cmd.run: - - name: {{ boussole_venv_dir }}/bin/boussole compile --backend=yaml --config=boussole.yaml - - cwd: {{ app_dir }} - - env: - - LC_ALL: C.UTF-8 - - LANG: C.UTF-8 - - require: - - file: create-css-directory - - unless: ls {{ app_dir }}/static/css/*.css diff --git a/salt/salt/cmark-gfm.sls b/salt/salt/cmark-gfm.sls deleted file mode 100644 index e0928b5..0000000 --- a/salt/salt/cmark-gfm.sls +++ /dev/null @@ -1,25 +0,0 @@ -unpack-cmark-gfm: - archive.extracted: - - name: /tmp/cmark-gfm - - source: - - salt://cmark-gfm.tar.gz - - https://github.com/github/cmark-gfm/archive/0.29.0.gfm.0.tar.gz - - source_hash: sha256=6a94aeaa59a583fadcbf28de81dea8641b3f56d935dda5b2447a3c8df6c95fea - - if_missing: /usr/local/lib/libcmark-gfm.so - - options: --strip-components=1 - - enforce_toplevel: False - -install-cmark-build-deps: - pkg.installed: - - name: cmake - -install-cmark-gfm: - cmd.run: - - cwd: /tmp/cmark-gfm/ - - names: - - make - - make install - - onchanges: - - archive: unpack-cmark-gfm - - require: - - pkg: install-cmark-build-deps diff --git a/salt/salt/common.jinja2 b/salt/salt/common.jinja2 deleted file mode 100644 index ee73a10..0000000 --- a/salt/salt/common.jinja2 +++ /dev/null @@ -1,13 +0,0 @@ -{% set python_version = '3.8' %} - -{% set app_dir = '/opt/tildes' %} -{% set static_sites_dir = '/opt/tildes-static-sites' %} - -{% set venv_dir = '/opt/venvs/tildes' %} -{% set bin_dir = venv_dir + '/bin' %} - -{% if grains['id'] == 'dev' %} - {% set app_username = 'vagrant' %} -{% else %} - {% set app_username = 'tildes' %} -{% endif %} diff --git a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 b/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 deleted file mode 100644 index 352eb76..0000000 --- a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Comment User Mention Generator (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python comment_user_mentions_generator.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/init.sls b/salt/salt/consumers/init.sls deleted file mode 100644 index 5c4df93..0000000 --- a/salt/salt/consumers/init.sls +++ /dev/null @@ -1,85 +0,0 @@ -/etc/systemd/system/consumer-topic_interesting_activity_updater.service: - file.managed: - - source: salt://consumers/topic_interesting_activity_updater.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-topic_metadata_generator.service: - file.managed: - - source: salt://consumers/topic_metadata_generator.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-comment_user_mentions_generator.service: - file.managed: - - source: salt://consumers/comment_user_mentions_generator.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/consumer-post_processing_script_runner.service: - file.managed: - - source: salt://consumers/post_processing_script_runner.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_interesting_activity_updater.service: - service.running: - - enable: True - -consumer-topic_metadata_generator.service: - service.running: - - enable: True - -consumer-comment_user_mentions_generator.service: - service.running: - - enable: True - -consumer-post_processing_script_runner.service: - service.running: - - enable: True - -{% if grains['id'] == 'prod' %} -/etc/systemd/system/consumer-topic_embedly_extractor.service: - file.managed: - - source: salt://consumers/topic_embedly_extractor.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_embedly_extractor.service: - service.running: - - enable: True - -/etc/systemd/system/consumer-topic_youtube_scraper.service: - file.managed: - - source: salt://consumers/topic_youtube_scraper.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-topic_youtube_scraper.service: - service.running: - - enable: True - -/etc/systemd/system/consumer-site_icon_downloader.service: - file.managed: - - source: salt://consumers/site_icon_downloader.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -consumer-site_icon_downloader.service: - service.running: - - enable: True -{% endif %} diff --git a/salt/salt/consumers/post_processing_script_runner.service.jinja2 b/salt/salt/consumers/post_processing_script_runner.service.jinja2 deleted file mode 100644 index b7c0f34..0000000 --- a/salt/salt/consumers/post_processing_script_runner.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Post Processing Script Runner (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python post_processing_script_runner.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/site_icon_downloader.service.jinja2 b/salt/salt/consumers/site_icon_downloader.service.jinja2 deleted file mode 100644 index ee7a98a..0000000 --- a/salt/salt/consumers/site_icon_downloader.service.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Site Icon Downloader (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python site_icon_downloader.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 b/salt/salt/consumers/topic_embedly_extractor.service.jinja2 deleted file mode 100644 index a0061ea..0000000 --- a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Embedly Extractor (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_embedly_extractor.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 b/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 deleted file mode 100644 index 0cf8fc5..0000000 --- a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Interesting Activity Updater (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_interesting_activity_updater.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_metadata_generator.service.jinja2 b/salt/salt/consumers/topic_metadata_generator.service.jinja2 deleted file mode 100644 index 0d20257..0000000 --- a/salt/salt/consumers/topic_metadata_generator.service.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} -[Unit] -Description=Topic Metadata Generator (Queue Consumer) -Requires=redis.service -After=redis.service -PartOf=redis.service - -[Service] -User={{ app_username }} -Group={{ app_username }} -WorkingDirectory={{ app_dir }}/consumers -Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" -ExecStart={{ bin_dir }}/python topic_metadata_generator.py -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/cronjobs.sls b/salt/salt/cronjobs.sls deleted file mode 100644 index 2d561ad..0000000 --- a/salt/salt/cronjobs.sls +++ /dev/null @@ -1,46 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir %} - -lift-expired-temp-bans-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.lift_expired_temporary_bans import lift_expired_temporary_bans; lift_expired_temporary_bans('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 1 - -close-voting-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.close_voting_on_old_posts import close_voting_on_old_posts; close_voting_on_old_posts('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 3 - -data-cleanup-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.clean_private_data import clean_all_data; clean_all_data('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - hour: 4 - - minute: 10 - -generate-group-stats-for-yesterday-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.generate_group_stats_for_yesterday import generate_stats; generate_stats('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: 0 - - minute: 10 - -generate-site-icons-css-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.generate_site_icons_css import generate_css; generate_css()" - - user: {{ app_username }} - - minute: '*/5' - -post-scheduled-topics-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.post_scheduled_topics import post_scheduled_topics; post_scheduled_topics('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - -update-common-topic-tags-cronjob: - cron.present: - - name: {{ bin_dir }}/python -c "from scripts.update_groups_common_topic_tags import update_common_topic_tags; update_common_topic_tags('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - user: {{ app_username }} - - hour: '*' - - minute: 0 diff --git a/salt/salt/development.sls b/salt/salt/development.sls deleted file mode 100644 index bc4776e..0000000 --- a/salt/salt/development.sls +++ /dev/null @@ -1,27 +0,0 @@ -{% from 'common.jinja2' import app_username, bin_dir %} - -{% set profile_dir = '/home/' + app_username + '/.ipython/profile_default' %} - -ipython-profile: - cmd.run: - - name: {{ bin_dir }}/ipython profile create - - runas: {{ app_username }} - - creates: {{ profile_dir }} - file.managed: - - name: {{ profile_dir }}/ipython_config.py - - contents: - - c.InteractiveShellApp.extensions = ['autoreload'] - - c.InteractiveShellApp.exec_lines = ['%autoreload 2'] - require: - - pip: pip-installs - -automatic-activate: - file.append: - - name: '/home/{{ app_username }}/.bashrc' - - text: 'source activate' - -# adds invoke's tab-completion script so that invoke tasks can be completed -invoke-tab-completion: - file.append: - - name: '/home/{{ app_username }}/.bashrc' - - text: 'source <(invoke --print-completion-script bash)' diff --git a/salt/salt/final-setup.sls b/salt/salt/final-setup.sls deleted file mode 100644 index 6f3cb48..0000000 --- a/salt/salt/final-setup.sls +++ /dev/null @@ -1,19 +0,0 @@ -{% from 'common.jinja2' import app_dir, bin_dir %} - -initialize-db: - cmd.run: - - name: {{ bin_dir }}/python -c "from scripts.initialize_db import initialize_db; initialize_db('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - cwd: {{ app_dir }} - - runas: postgres - - require: - - postgres_database: tildes - - unless: psql -U tildes tildes -c "SELECT user_id FROM users;" - -{% if grains['id'] == 'dev' %} -insert-dev-data: - cmd.run: - - name: {{ bin_dir }}/python -c "from scripts.initialize_db import insert_dev_data; insert_dev_data('{{ app_dir }}/{{ pillar['ini_file'] }}')" - - cwd: {{ app_dir }} - - onchanges: - - cmd: initialize-db -{% endif %} diff --git a/salt/salt/general-config.sls b/salt/salt/general-config.sls deleted file mode 100644 index 22c8f68..0000000 --- a/salt/salt/general-config.sls +++ /dev/null @@ -1,3 +0,0 @@ -set-timezone-to-utc: - timezone.system: - - name: 'Etc/UTC' diff --git a/salt/salt/grafana/grafana.conf.jinja2 b/salt/salt/grafana/grafana.conf.jinja2 deleted file mode 100644 index 480d59a..0000000 --- a/salt/salt/grafana/grafana.conf.jinja2 +++ /dev/null @@ -1,19 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['grafana_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_redirect off; - proxy_pass http://localhost:3000; - } -} diff --git a/salt/salt/grafana/grafana.ini.jinja2 b/salt/salt/grafana/grafana.ini.jinja2 deleted file mode 100644 index db942d8..0000000 --- a/salt/salt/grafana/grafana.ini.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -[server] -domain = {{ pillar['grafana_server_name'] }} -root_url = https://{{ pillar['grafana_server_name'] }} - -[analytics] -reporting_enabled = false - -[security] -admin_password = """{{ pillar['grafana_admin_password'] }}""" -disable_gravatar = true - -[snapshots] -external_enabled = false - -[users] -allow_sign_up = false -allow_org_create = false diff --git a/salt/salt/grafana/init.sls b/salt/salt/grafana/init.sls deleted file mode 100644 index 7c556bd..0000000 --- a/salt/salt/grafana/init.sls +++ /dev/null @@ -1,61 +0,0 @@ -grafana: - pkgrepo.managed: - - name: deb https://packages.grafana.com/oss/deb stable main - - dist: stable - - file: /etc/apt/sources.list.d/grafana.list - - key_url: https://packages.grafana.com/gpg.key - - require_in: - - pkg: grafana - pkg.installed: - - name: grafana - - refresh: True - # note: this file must be set up before the server is started for the first - # time, otherwise the admin password will not be set correctly - file.managed: - - name: /etc/grafana/grafana.ini - - source: salt://grafana/grafana.ini.jinja2 - - template: jinja - - user: root - - group: grafana - - mode: 640 - service.running: - - name: grafana-server - - enable: True - - require: - - pkg: grafana - - file: /etc/grafana/grafana.ini - - watch: - - file: /etc/grafana/grafana.ini - http.query: - - name: http://localhost:3000/api/datasources - - method: POST - - username: admin - - password: {{ pillar['grafana_admin_password'] }} - - data: | - {"name": "Prometheus", - "type": "prometheus", - "url": "http://localhost:9090", - "access": "proxy", - "isDefault": true} - - header_list: - - 'Content-Type: application/json' - - status: 200 - - unless: - - curl -f http://admin:{{ pillar['grafana_admin_password'] }}@localhost:3000/api/datasources/name/Prometheus - -/etc/nginx/sites-available/grafana.conf: - file.managed: - - source: salt://grafana/grafana.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/grafana.conf: - file.symlink: - - target: /etc/nginx/sites-available/grafana.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/gunicorn/init.sls b/salt/salt/gunicorn/init.sls deleted file mode 100644 index bcaa488..0000000 --- a/salt/salt/gunicorn/init.sls +++ /dev/null @@ -1,59 +0,0 @@ -{% from 'common.jinja2' import app_username %} - -/etc/systemd/system/gunicorn.service: - file.managed: - - source: salt://gunicorn/gunicorn.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -/etc/systemd/system/gunicorn.socket: - file.managed: - - source: salt://gunicorn/gunicorn.socket.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -/usr/lib/tmpfiles.d/gunicorn.conf: - file.managed: - - source: salt://gunicorn/gunicorn.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn.socket - -gunicorn.socket: - service.running: - - enable: True - -# Set up the gunicorn_reloader service, which reloads gunicorn whenever certain files -# are changed (such as static files, to update the cache-busting strings) -/etc/systemd/system/gunicorn_reloader.service: - file.managed: - - source: salt://gunicorn/gunicorn_reloader.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn_reloader.path - -/etc/systemd/system/gunicorn_reloader.path: - file.managed: - - source: salt://gunicorn/gunicorn_reloader.path - - user: root - - group: root - - mode: 644 - - require_in: - - service: gunicorn_reloader.path - -gunicorn_reloader.path: - service.running: - - enable: True diff --git a/salt/salt/java.sls b/salt/salt/java.sls deleted file mode 100644 index 2faa64f..0000000 --- a/salt/salt/java.sls +++ /dev/null @@ -1,3 +0,0 @@ -java-openjdk: - pkg.installed: - - name: openjdk-8-jre diff --git a/salt/salt/nginx/init.sls b/salt/salt/nginx/init.sls deleted file mode 100644 index 1b84a6d..0000000 --- a/salt/salt/nginx/init.sls +++ /dev/null @@ -1,33 +0,0 @@ -nginx: - pkgrepo.managed: - - name: deb http://nginx.org/packages/ubuntu/ xenial nginx - - dist: xenial - - file: /etc/apt/sources.list.d/nginx.list - - key_url: https://nginx.org/keys/nginx_signing.key - - require_in: - - pkg: nginx - pkg.installed: - - name: nginx - - refresh: True - service.running: - - require: - - pkg: nginx - - reload: True - - watch: - - file: /etc/nginx/* - -/etc/nginx/nginx.conf: - file.managed: - - source: salt://nginx/nginx.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -# Add logrotate config to rotate daily and delete after 30 days -/etc/logrotate.d/nginx: - file.managed: - - source: salt://nginx/logrotate - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/shortener-config.sls b/salt/salt/nginx/shortener-config.sls deleted file mode 100644 index 03cc034..0000000 --- a/salt/salt/nginx/shortener-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes-shortener.conf: - file.managed: - - source: salt://nginx/tildes-shortener.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes-shortener.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes-shortener.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/site-config.sls b/salt/salt/nginx/site-config.sls deleted file mode 100644 index e42e420..0000000 --- a/salt/salt/nginx/site-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes.conf: - file.managed: - - source: salt://nginx/tildes.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nginx/static-sites-config.sls b/salt/salt/nginx/static-sites-config.sls deleted file mode 100644 index 2d034b6..0000000 --- a/salt/salt/nginx/static-sites-config.sls +++ /dev/null @@ -1,16 +0,0 @@ -/etc/nginx/sites-available/tildes-static-sites.conf: - file.managed: - - source: salt://nginx/tildes-static-sites.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/tildes-static-sites.conf: - file.symlink: - - target: /etc/nginx/sites-available/tildes-static-sites.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/nodejs.sls b/salt/salt/nodejs.sls deleted file mode 100644 index b6ab35f..0000000 --- a/salt/salt/nodejs.sls +++ /dev/null @@ -1,25 +0,0 @@ -{% from "common.jinja2" import app_dir, app_username %} - -# Add the NodeSource repository and install Node.js 10.x -nodejs-pkgrepo: - pkgrepo.managed: - - name: deb https://deb.nodesource.com/node_10.x xenial main - - dist: xenial - - file: /etc/apt/sources.list.d/nodesource.list - - key_url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key - - require_in: - - pkg: nodejs - pkg.installed: - - name: nodejs - - refresh: True - -# Install the npm packages defined in package.json -# Uses the --no-bin-links option to prevent npm from creating symlinks in the .bin -# directory, which doesn't work inside Vagrant on Windows -install-npm-packages: - cmd.run: - - name: npm install --no-bin-links - - cwd: {{ app_dir }} - - runas: {{ app_username }} - - require: - - pkg: nodejs diff --git a/salt/salt/postgresql-redis-bridge.sls b/salt/salt/postgresql-redis-bridge.sls deleted file mode 100644 index 411e6cd..0000000 --- a/salt/salt/postgresql-redis-bridge.sls +++ /dev/null @@ -1,14 +0,0 @@ -/etc/systemd/system/postgresql_redis_bridge.service: - file.managed: - - source: salt://postgresql_redis_bridge.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require: - - service: redis.service - - service: postgresql - -postgresql_redis_bridge.service: - service.running: - - enable: True diff --git a/salt/salt/postgresql/init.sls b/salt/salt/postgresql/init.sls deleted file mode 100644 index b251d81..0000000 --- a/salt/salt/postgresql/init.sls +++ /dev/null @@ -1,85 +0,0 @@ -{% from 'common.jinja2' import app_dir, python_version, venv_dir %} - -postgresql: - pkgrepo.managed: - - name: deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main - - dist: xenial-pgdg - - file: /etc/apt/sources.list.d/psql.list - - key_url: https://www.postgresql.org/media/keys/ACCC4CF8.asc - - require_in: - - pkg: postgresql - pkg.installed: - - name: postgresql-{{ pillar['postgresql_version'] }} - - refresh: True - service.running: - - require: - - pkg: postgresql - - reload: True - - watch: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/*.conf - -install-plpython3: - pkg.installed: - - name: postgresql-plpython3-{{ pillar['postgresql_version'] }} - - refresh: True - - require: - - pkgrepo: postgresql - -set-lock-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?lock_timeout = (?!5000).*$' - - repl: 'lock_timeout = 5000' - -set-statement-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?statement_timeout = (?!5000).*$' - - repl: 'statement_timeout = 5000' - -set-idle-in-transaction-timeout: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?idle_in_transaction_session_timeout = (?!600000).*$' - - repl: 'idle_in_transaction_session_timeout = 600000' - -set-timezone: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?timezone = (?!''UTC'').*$' - - repl: 'timezone = ''UTC''' - -enable-pg-stat-statements: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?shared_preload_libraries = .*$' - - repl: "shared_preload_libraries = 'pg_stat_statements'" - - watch_in: - - module: enable-pg-stat-statements - module.wait: - - service.restart: - - name: postgresql.service - -/etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf: - file.managed: - - source: salt://postgresql/pg_hba.conf.jinja2 - - template: jinja - - user: postgres - - group: postgres - - mode: 640 - require: - - service: postgresql - -# set PYTHONPATH env var in postgresql so PL/Python can access all the modules -set-postgresql-pythonpath: - file.managed: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/environment - - contents: "PYTHONPATH='{{ venv_dir }}/lib/python{{ python_version }}/site-packages:{{ app_dir }}'" - - user: postgres - - group: postgres - - mode: 644 - - watch_in: - - module: set-postgresql-pythonpath - module.wait: - - service.restart: - - name: postgresql.service diff --git a/salt/salt/postgresql/pg_hba.conf.jinja2 b/salt/salt/postgresql/pg_hba.conf.jinja2 deleted file mode 100644 index 45e7ba9..0000000 --- a/salt/salt/postgresql/pg_hba.conf.jinja2 +++ /dev/null @@ -1,6 +0,0 @@ -{% for line in accumulator['pg_hba_lines'] -%} -{{ line }} -{% endfor %} - -# Required: allow the database superuser to connect via socket -local all postgres peer diff --git a/salt/salt/postgresql/pgbouncer.sls b/salt/salt/postgresql/pgbouncer.sls deleted file mode 100644 index 21d7453..0000000 --- a/salt/salt/postgresql/pgbouncer.sls +++ /dev/null @@ -1,25 +0,0 @@ -install-pgbouncer: - pkg.installed: - - name: pgbouncer - -/etc/pgbouncer/pgbouncer.ini: - file.managed: - - source: salt://postgresql/pgbouncer.ini.jinja2 - - template: jinja - - user: postgres - - group: postgres - - mode: 640 - -/etc/pgbouncer/userlist.txt: - file.managed: - - contents: '"tildes" ""' - - user: postgres - - group: postgres - - mode: 640 - -pgbouncer.service: - service.running: - - enable: True - - reload: True - - watch: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/postgresql/prod-config.sls b/salt/salt/postgresql/prod-config.sls deleted file mode 100644 index 40c477d..0000000 --- a/salt/salt/postgresql/prod-config.sls +++ /dev/null @@ -1,25 +0,0 @@ -# Values from PGTune (https://pgtune.leopard.in.ua/) -{% set setting_values = { - "checkpoint_completion_target": "0.7", - "default_statistics_target": "100", - "effective_cache_size": "24GB", - "effective_io_concurrency": 200, - "maintenance_work_mem": "2GB", - "max_parallel_workers": "8", - "max_parallel_workers_per_gather": "4", - "max_wal_size": "2GB", - "max_worker_processes": "8", - "min_wal_size": "1GB", - "random_page_cost": "1.1", - "shared_buffers": "8GB", - "wal_buffers": "16MB", - "work_mem": "10485kB", -} %} - -{% for setting, value in setting_values.items() %} -postgresql-conf-set-{{ setting }}: - file.replace: - - name: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/postgresql.conf - - pattern: '^#?{{ setting }} = (?!{{ value }}).*$' - - repl: '{{ setting }} = {{ value }}' -{% endfor %} diff --git a/salt/salt/postgresql/site-db.sls b/salt/salt/postgresql/site-db.sls deleted file mode 100644 index 52b0fef..0000000 --- a/salt/salt/postgresql/site-db.sls +++ /dev/null @@ -1,73 +0,0 @@ -site-db-user: - postgres_user.present: - - name: tildes - # set the tildes user to a database superuser in the dev environment only - - superuser: {{ grains['id'] == 'dev' }} - - require: - - service: postgresql - -site-db-database: - postgres_database.present: - - name: tildes - - owner: tildes - - require: - - postgres_user: tildes - -site-db-enable-citext: - postgres_extension.present: - - name: citext - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-ltree: - postgres_extension.present: - - name: ltree - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-intarray: - postgres_extension.present: - - name: intarray - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-pg_stat_statements: - postgres_extension.present: - - name: pg_stat_statements - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-pg_trgm: - postgres_extension.present: - - name: pg_trgm - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-enable-plpython3u: - postgres_extension.present: - - name: plpython3u - - maintenance_db: tildes - - require: - - postgres_database: tildes - -site-db-pg_hba-lines: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: - - 'local sameuser tildes trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -site-db-pgbouncer-lines: - file.accumulated: - - name: pgbouncer_db_lines - - filename: /etc/pgbouncer/pgbouncer.ini - - text: 'tildes =' - - require_in: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/postgresql/test-db.sls b/salt/salt/postgresql/test-db.sls deleted file mode 100644 index 9f868d9..0000000 --- a/salt/salt/postgresql/test-db.sls +++ /dev/null @@ -1,58 +0,0 @@ -test-db-database: - postgres_database.present: - - name: tildes_test - - owner: tildes - - require: - - postgres_user: tildes - -test-db-enable-citext: - postgres_extension.present: - - name: citext - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-ltree: - postgres_extension.present: - - name: ltree - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-intarray: - postgres_extension.present: - - name: intarray - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-pg_trgm: - postgres_extension.present: - - name: pg_trgm - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-enable-plpython3u: - postgres_extension.present: - - name: plpython3u - - maintenance_db: tildes_test - - require: - - postgres_database: tildes_test - -test-db-pg_hba-lines: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: - - 'local tildes_test tildes trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -test-db-pgbouncer-lines: - file.accumulated: - - name: pgbouncer_db_lines - - filename: /etc/pgbouncer/pgbouncer.ini - - text: 'tildes_test =' - - require_in: - - file: /etc/pgbouncer/pgbouncer.ini diff --git a/salt/salt/prod-config.sls b/salt/salt/prod-config.sls deleted file mode 100644 index 3e1ee10..0000000 --- a/salt/salt/prod-config.sls +++ /dev/null @@ -1,17 +0,0 @@ -# Force apt not to use IPv6 (seems to hang often) -/etc/apt/apt.conf.d/99force-ipv4: - file.managed: - - contents: 'Acquire::ForceIPv4 "true";' - -# Enable ipv6 networking -/etc/network/interfaces: - file.append: - - text: | - iface eth0 inet6 static - address {{ pillar['ipv6_address'] }} - netmask 64 - - post-up sleep 5; /sbin/ip -family inet6 route add {{ pillar['ipv6_gateway'] }} dev eth0 - post-up sleep 5; /sbin/ip -family inet6 route add default via {{ pillar['ipv6_gateway'] }} - pre-down /sbin/ip -family inet6 route del default via {{ pillar['ipv6_gateway'] }} - pre-down /sbin/ip -family inet6 route del {{ pillar['ipv6_gateway'] }} dev eth0 diff --git a/salt/salt/prometheus/exporters/blackbox.yml b/salt/salt/prometheus/exporters/blackbox.yml deleted file mode 100644 index 33cf2cb..0000000 --- a/salt/salt/prometheus/exporters/blackbox.yml +++ /dev/null @@ -1,14 +0,0 @@ -modules: - site_ipv4: - prober: http - timeout: 5s - http: - preferred_ip_protocol: "ip4" - ip_protocol_fallback: false - - site_ipv6: - prober: http - timeout: 5s - http: - preferred_ip_protocol: "ip6" - ip_protocol_fallback: false diff --git a/salt/salt/prometheus/exporters/blackbox_exporter.sls b/salt/salt/prometheus/exporters/blackbox_exporter.sls deleted file mode 100644 index 42884b8..0000000 --- a/salt/salt/prometheus/exporters/blackbox_exporter.sls +++ /dev/null @@ -1,35 +0,0 @@ -# Download/extract and set up the blackbox exporter -include: - - prometheus.user - -unpack-blackbox-exporter: - archive.extracted: - - name: /opt/prometheus_blackbox_exporter - - source: - - salt://prometheus/exporters/blackbox_exporter-0.16.0.linux-amd64.tar.gz - - https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz - - source_hash: sha256=52d3444a518ea01f220e08eaa53eb717ef54da6724760c925ab41285d0d5a7bd - - if_missing: /opt/prometheus_blackbox_exporter - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/opt/prometheus_blackbox_exporter/blackbox.yml: - file.managed: - - source: salt://prometheus/exporters/blackbox.yml - - user: prometheus - - group: prometheus - - mode: 644 - -/etc/systemd/system/prometheus_blackbox_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_blackbox_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-blackbox-exporter-service: - service.running: - - name: prometheus_blackbox_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/node_exporter.sls b/salt/salt/prometheus/exporters/node_exporter.sls deleted file mode 100644 index 06d41b8..0000000 --- a/salt/salt/prometheus/exporters/node_exporter.sls +++ /dev/null @@ -1,28 +0,0 @@ -# Download/extract and set up the node exporter (hardware/OS metrics) -include: - - prometheus.user - -unpack-node-exporter: - archive.extracted: - - name: /opt/prometheus_node_exporter - - source: - - salt://prometheus/exporters/node_exporter-0.13.0.linux-amd64.tar.gz - - https://github.com/prometheus/node_exporter/releases/download/v0.13.0/node_exporter-0.13.0.linux-amd64.tar.gz - - source_hash: sha256=2de5d1e51330c41588ed4c88bc531a3d2dccf6b4d7b99d5782d95cff27a3c049 - - if_missing: /opt/prometheus_node_exporter - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/etc/systemd/system/prometheus_node_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_node_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-node-exporter-service: - service.running: - - name: prometheus_node_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/postgres_exporter.sls b/salt/salt/prometheus/exporters/postgres_exporter.sls deleted file mode 100644 index 9c22869..0000000 --- a/salt/salt/prometheus/exporters/postgres_exporter.sls +++ /dev/null @@ -1,36 +0,0 @@ -# Download and set up the postgres exporter -include: - - prometheus.user - -unpack-postgres-exporter: - archive.extracted: - - name: /opt/prometheus_postgres_exporter - - source: - - salt://prometheus/exporters/postgres_exporter_v0.4.7_linux-amd64.tar.gz - - https://github.com/wrouesnel/postgres_exporter/releases/download/v0.4.7/postgres_exporter_v0.4.7_linux-amd64.tar.gz - - source_hash: sha256=c34d61bb4deba8efae06fd3c9979b96dae3f3c757698ce3384c80fff586c667b - - if_missing: /opt/prometheus_postgres_exporter - - user: postgres - - group: postgres - - mode: 774 - - options: --strip-components=1 - - enforce_toplevel: False - -/opt/prometheus_postgres_exporter/queries.yaml: - file.managed: - - source: salt://prometheus/exporters/queries.yaml - - user: postgres - - group: postgres - - mode: 644 - -/etc/systemd/system/prometheus_postgres_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_postgres_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-postgres-exporter-service: - service.running: - - name: prometheus_postgres_exporter - - enable: True diff --git a/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service b/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service deleted file mode 100644 index 430c308..0000000 --- a/salt/salt/prometheus/exporters/prometheus_blackbox_exporter.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Prometheus Blackbox Exporter -After=syslog.target network.target - -[Service] -Type=simple -RemainAfterExit=no -WorkingDirectory=/opt/prometheus_blackbox_exporter -User=prometheus -Group=prometheus -ExecStart=/opt/prometheus_blackbox_exporter/blackbox_exporter --config.file=/opt/prometheus_blackbox_exporter/blackbox.yml - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/prometheus/exporters/redis_exporter.sls b/salt/salt/prometheus/exporters/redis_exporter.sls deleted file mode 100644 index a731283..0000000 --- a/salt/salt/prometheus/exporters/redis_exporter.sls +++ /dev/null @@ -1,27 +0,0 @@ -# Download/extract and set up the redis exporter -include: - - prometheus.user - -unpack-redis-exporter: - archive.extracted: - - name: /opt/prometheus_redis_exporter - - source: - - salt://prometheus/exporters/redis_exporter-v0.26.0.linux-amd64.tar.gz - - https://github.com/oliver006/redis_exporter/releases/download/v0.26.0/redis_exporter-v0.26.0.linux-amd64.tar.gz - - source_hash: sha256=39354c57b9d02b455c584baf46a2df6ed3d1ac190c88e3ec0fa0c23b49ccc2bb - - if_missing: /opt/prometheus_redis_exporter - - user: prometheus - - group: prometheus - - enforce_toplevel: False - -/etc/systemd/system/prometheus_redis_exporter.service: - file.managed: - - source: salt://prometheus/exporters/prometheus_redis_exporter.service - - user: root - - group: root - - mode: 644 - -prometheus-redis-exporter-service: - service.running: - - name: prometheus_redis_exporter - - enable: True diff --git a/salt/salt/prometheus/init.sls b/salt/salt/prometheus/init.sls deleted file mode 100644 index 5001fe2..0000000 --- a/salt/salt/prometheus/init.sls +++ /dev/null @@ -1,58 +0,0 @@ -include: - - prometheus.user - -unpack-prometheus: - archive.extracted: - - name: /opt/prometheus - - source: - - salt://prometheus/prometheus-2.0.0.linux-amd64.tar.gz - - https://github.com/prometheus/prometheus/releases/download/v2.0.0/prometheus-2.0.0.linux-amd64.tar.gz - - source_hash: sha256=e12917b25b32980daee0e9cf879d9ec197e2893924bd1574604eb0f550034d46 - - if_missing: /opt/prometheus - - user: prometheus - - group: prometheus - - options: --strip-components=1 - - enforce_toplevel: False - -/etc/systemd/system/prometheus.service: - file.managed: - - source: salt://prometheus/prometheus.service - - user: root - - group: root - - mode: 644 - -prometheus-service: - service.running: - - name: prometheus - - enable: True - - watch: - - file: /opt/prometheus/prometheus.yml - - -/opt/prometheus/prometheus.yml: - file.managed: - - source: salt://prometheus/prometheus.yml.jinja2 - - template: jinja - - user: prometheus - - group: prometheus - - mode: 664 - -{# Set up nginx to reverse-proxy to prometheus in production #} -{% if grains['id'] == 'prod' %} -/etc/nginx/sites-available/prometheus.conf: - file.managed: - - source: salt://prometheus/prometheus.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/prometheus.conf: - file.symlink: - - target: /etc/nginx/sites-available/prometheus.conf - - makedirs: True - - user: root - - group: root - - mode: 644 -{% endif %} diff --git a/salt/salt/prometheus/prometheus.conf.jinja2 b/salt/salt/prometheus/prometheus.conf.jinja2 deleted file mode 100644 index 3b4134f..0000000 --- a/salt/salt/prometheus/prometheus.conf.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['prometheus_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - proxy_pass http://localhost:9090; - } -} diff --git a/salt/salt/prometheus/user.sls b/salt/salt/prometheus/user.sls deleted file mode 100644 index 46dcf72..0000000 --- a/salt/salt/prometheus/user.sls +++ /dev/null @@ -1,7 +0,0 @@ -prometheus-user: - group.present: - - name: prometheus - user.present: - - name: prometheus - - groups: [prometheus] - - createhome: False diff --git a/salt/salt/pts-lbsearch.sls b/salt/salt/pts-lbsearch.sls deleted file mode 100644 index d09686a..0000000 --- a/salt/salt/pts-lbsearch.sls +++ /dev/null @@ -1,12 +0,0 @@ -compile-pts-lbsearch: - file.managed: - - name: /tmp/pts_lbsearch.c - - source: - - https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c - - source_hash: sha256=ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1 - - creates: /usr/local/bin/pts_lbsearch - cmd.run: - - cwd: /tmp/ - # compilation command taken from the top of the source file - - name: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c - - creates: /usr/local/bin/pts_lbsearch diff --git a/salt/salt/python.sls b/salt/salt/python.sls deleted file mode 100644 index 5626bf8..0000000 --- a/salt/salt/python.sls +++ /dev/null @@ -1,60 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username, bin_dir, python_version, venv_dir %} - -deadsnakes: - pkgrepo.managed: - - ppa: deadsnakes/ppa - pkg.installed: - - name: python{{ python_version }} - - refresh: True - -/opt/venvs: - file.directory: - - user: {{ app_username }} - - group: {{ app_username }} - - dir_mode: 755 - -delete-obsolete-venv: - file.absent: - - name: {{ venv_dir }} - - unless: {{ bin_dir }}/python --version | grep {{ python_version }} - -# Salt seems to use the deprecated pyvenv script, manual for now -venv-setup: - pkg.installed: - - name: python{{ python_version }}-venv - cmd.run: - - name: python{{ python_version }} -m venv {{ venv_dir }} - - creates: {{ venv_dir }} - - runas: {{ app_username }} - - require: - - pkg: python{{ python_version }}-venv - -# Packages needed to be able to compile psycopg2 (while installing requirements.txt) -psycopg2-deps: - pkg.installed: - - pkgs: - - gcc - - libpq-dev - - python{{ python_version }}-dev - -pip-installs: - pip.installed: - {% if grains['id'] == 'dev' %} - - requirements: {{ app_dir }}/requirements-dev.txt - {% else %} - - requirements: {{ app_dir }}/requirements.txt - {% endif %} - - user: {{ app_username }} - - bin_env: {{ venv_dir }} - require: - - cmd: venv-setup - - pkg: psycopg2-deps - -self-install: - pip.installed: - - bin_env: {{ venv_dir }} - - editable: - - {{ app_dir }} - - require: - - cmd: venv-setup - - unless: ls {{ venv_dir }}/lib/python{{ python_version }}/site-packages/tildes.egg-link diff --git a/salt/salt/redis/init.sls b/salt/salt/redis/init.sls deleted file mode 100644 index 8df19b7..0000000 --- a/salt/salt/redis/init.sls +++ /dev/null @@ -1,118 +0,0 @@ -{% set redis_version = '5.0.7' %} - -unpack-redis: - archive.extracted: - - name: /tmp/redis-{{ redis_version }} - - source: - - salt://redis/{{ redis_version }}.tar.gz - - https://github.com/antirez/redis/archive/{{ redis_version }}.tar.gz - - source_hash: sha256=2761422599f8969559e66797cd7f606c16e907bf82d962345a7d366c5d1278df - - unless: /usr/local/bin/redis-server --version | grep v={{ redis_version }} - - options: --strip-components=1 - - enforce_toplevel: False - -install-redis: - pkg.installed: - - pkgs: - - build-essential - cmd.run: - - cwd: /tmp/redis-{{ redis_version }}/ - - names: - - make - - make install - - onchanges: - - archive: unpack-redis - -redis-user: - group.present: - - name: redis - user.present: - - name: redis - - groups: [redis] - - createhome: False - -/run/redis: - file.directory: - - user: redis - - group: redis - - mode: 755 - - require: - - user: redis-user - -/var/lib/redis: - file.directory: - - user: redis - - group: redis - - mode: 700 - - require: - - user: redis-user - -/var/log/redis: - file.directory: - - user: redis - - group: redis - - mode: 744 - - require: - - user: redis-user - -/etc/redis.conf: - file.managed: - - source: salt://redis/redis.conf.jinja2 - - template: jinja - - user: redis - - group: redis - - mode: 600 - - require: - - user: redis-user - -/etc/systemd/system/redis.service: - file.managed: - - source: salt://redis/redis.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: redis.service - -# add the service file for disabling transparent hugepage -/etc/systemd/system/transparent_hugepage.service: - file.managed: - - source: salt://redis/transparent_hugepage.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: disable-transparent-hugepage - -# enable the "disable transparent hugepage" service and run it -disable-transparent-hugepage: - service.enabled: - - name: transparent_hugepage.service - cmd.run: - - name: systemctl start transparent_hugepage.service - - unless: 'cat /sys/kernel/mm/transparent_hugepage/enabled | grep \\[never\\]' - - require_in: - - service: redis.service - - watch_in: - - service: redis.service - -# Set kernel overcommit mode (recommended for Redis) -overcommit-memory: - # will take effect immediately - cmd.run: - - name: sysctl vm.overcommit_memory=1 - - unless: sysctl -n vm.overcommit_memory | grep 1 - - # makes the setting permanent but requires a restart - file.append: - - name: /etc/sysctl.conf - - text: 'vm.overcommit_memory = 1' - -redis.service: - service.running: - - enable: True - - watch: - - file: /etc/redis.conf - - require: - - user: redis-user - - cmd: install-redis diff --git a/salt/salt/redis/modules/redis-cell.sls b/salt/salt/redis/modules/redis-cell.sls deleted file mode 100644 index 6951b32..0000000 --- a/salt/salt/redis/modules/redis-cell.sls +++ /dev/null @@ -1,20 +0,0 @@ -redis-cell-unpack: - archive.extracted: - - name: /opt/redis-cell - - source: - - salt://redis/modules/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz - - https://github.com/brandur/redis-cell/releases/download/v0.2.1/redis-cell-v0.2.1-x86_64-unknown-linux-gnu.tar.gz - - source_hash: sha256=9427fb100f4cada817f30f854ead7f233de32948a0ec644f15988c275a2ed1cb - - if_missing: /opt/redis-cell - - enforce_toplevel: False - - require_in: - - service: redis.service - -redis-cell-loadmodule-line: - file.accumulated: - - name: redis_loadmodule_lines - - filename: /etc/redis.conf - - text: - - 'loadmodule /opt/redis-cell/libredis_cell.so' - - require_in: - - file: /etc/redis.conf diff --git a/salt/salt/scripts/init.sls b/salt/salt/scripts/init.sls deleted file mode 100644 index 2db61e7..0000000 --- a/salt/salt/scripts/init.sls +++ /dev/null @@ -1,7 +0,0 @@ -/usr/local/bin/activate: - file.managed: - - source: salt://scripts/activate.sh.jinja2 - - template: jinja - - user: root - - group: root - - mode: 755 diff --git a/salt/salt/self-signed-cert.sls b/salt/salt/self-signed-cert.sls deleted file mode 100644 index f1108c1..0000000 --- a/salt/salt/self-signed-cert.sls +++ /dev/null @@ -1,17 +0,0 @@ -self-signed-cert: - pkg.installed: - - name: python3-openssl - module.run: - - tls.create_self_signed_cert: - - days: 3650 - - require: - - pkg: python3-openssl - - unless: ls /etc/pki/tls/certs/localhost.key - file.managed: - - name: /etc/pki/tls/certs/localhost.key - - mode: 600 - - replace: False - - require: - - module: self-signed-cert - - require_in: - - service: nginx diff --git a/salt/salt/sentry/common.jinja2 b/salt/salt/sentry/common.jinja2 deleted file mode 100644 index 055cf3d..0000000 --- a/salt/salt/sentry/common.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ -{% set sentry_cfg_dir = '/etc/sentry' %} - -{% set sentry_venv_dir = '/opt/venvs/sentry' %} -{% set sentry_bin_dir = sentry_venv_dir + '/bin' %} diff --git a/salt/salt/sentry/config.yml.jinja2 b/salt/salt/sentry/config.yml.jinja2 deleted file mode 100644 index d78a880..0000000 --- a/salt/salt/sentry/config.yml.jinja2 +++ /dev/null @@ -1,64 +0,0 @@ -# While a lot of configuration in Sentry can be changed via the UI, for all -# new-style config (as of 8.0) you can also declare values here in this file -# to enforce defaults or to ensure they cannot be changed via the UI. For more -# information see the Sentry documentation. - -############### -# Mail Server # -############### - -mail.backend: 'dummy' # Use dummy if you want to disable email entirely -# mail.host: 'localhost' -# mail.port: 25 -# mail.username: '' -# mail.password: '' -# mail.use-tls: false -# The email address to send on behalf of -# mail.from: 'root@localhost' - -# If you'd like to configure email replies, enable this. -# mail.enable-replies: false - -# When email-replies are enabled, this value is used in the Reply-To header -# mail.reply-hostname: '' - -# If you're using mailgun for inbound mail, set your API key and configure a -# route to forward to /api/hooks/mailgun/inbound/ -# mail.mailgun-api-key: '' - -################### -# System Settings # -################### - -# If this file ever becomes compromised, it's important to regenerate your a new key -# Changing this value will result in all current sessions being invalidated. -# A new key can be generated with `$ sentry config generate-secret-key` -system.secret-key: '{{ pillar['sentry_secret'] }}' - -# The ``redis.clusters`` setting is used, unsurprisingly, to configure Redis -# clusters. These clusters can be then referred to by name when configuring -# backends such as the cache, digests, or TSDB backend. -redis.clusters: - default: - hosts: - 0: - host: 127.0.0.1 - port: 6379 - db: 10 - -################ -# File storage # -################ - -# Uploaded media uses these `filestore` settings. The available -# backends are either `filesystem` or `s3`. - -filestore.backend: 'filesystem' -filestore.options: - location: '/tmp/sentry-files' - -# filestore.backend: 's3' -# filestore.options: -# access_key: 'AKIXXXXXX' -# secret_key: 'XXXXXXX' -# bucket_name: 's3-bucket-name' diff --git a/salt/salt/sentry/init.sls b/salt/salt/sentry/init.sls deleted file mode 100644 index 4c69500..0000000 --- a/salt/salt/sentry/init.sls +++ /dev/null @@ -1,148 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir %} - -sentry-user: - group.present: - - name: sentry - user.present: - - name: sentry - - groups: [sentry] - -build-deps: - pkg.installed: - - pkgs: - - python-setuptools - - python-dev - - python-pip - - python-virtualenv - - libxslt1-dev - - gcc - - libffi-dev - - libjpeg-dev - - libxml2-dev - - libxslt-dev - - libyaml-dev - - libpq-dev - - zlib1g-dev - - postgresql-server-dev-{{ pillar['postgresql_version'] }} - -{{ sentry_venv_dir }}: - virtualenv.managed: - - system_site_packages: False - -pip-install: - pip.installed: - - name: sentry - - bin_env: {{ sentry_venv_dir }} - -pip-install-plugins: - pip.installed: - - name: sentry-plugins - - bin_env: {{ sentry_venv_dir }} - -init-sentry: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry init {{ sentry_cfg_dir }} - - creates: {{ sentry_cfg_dir }} - -postgres-setup: - postgres_user.present: - - name: sentry - postgres_database.present: - - name: sentry - - owner: sentry - # sentry migrations should add this, but it fails due to not being superuser - postgres_extension.present: - - name: citext - - maintenance_db: sentry - -{{ sentry_cfg_dir }}/sentry.conf.py: - file.managed: - - source: salt://sentry/sentry.conf.py - -{{ sentry_cfg_dir }}/config.yml: - file.managed: - - source: salt://sentry/config.yml.jinja2 - - template: jinja - -update-pg_hba: - file.accumulated: - - name: pg_hba_lines - - filename: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - - text: 'host sameuser sentry 127.0.0.1/32 trust' - - require_in: - - file: /etc/postgresql/{{ pillar['postgresql_version'] }}/main/pg_hba.conf - -create-sentry-db: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry upgrade --noinput - - env: - - SENTRY_CONF: '{{ sentry_cfg_dir }}' - - onchanges: - - cmd: init-sentry - -create-sentry-user: - cmd.run: - - name: {{ sentry_bin_dir }}/sentry createuser --no-input --email {{ pillar['sentry_email'] }} --password {{ pillar['sentry_password'] }} - - env: - - SENTRY_CONF: '{{ sentry_cfg_dir }}' - - onchanges: - - cmd: init-sentry - -/etc/systemd/system/sentry-web.service: - file.managed: - - source: salt://sentry/sentry-web.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/sentry-worker.service: - file.managed: - - source: salt://sentry/sentry-worker.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -/etc/systemd/system/sentry-cron.service: - file.managed: - - source: salt://sentry/sentry-cron.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - -sentry-web: - service.running: - - enable: True - -sentry-worker: - service.running: - - enable: True - -sentry-cron: - service.running: - - enable: True - -sentry-cleanup: - cron.present: - - name: {{ sentry_bin_dir }}/sentry cleanup --days=30 - - hour: 4 - - minute: 0 - -/etc/nginx/sites-available/sentry.conf: - file.managed: - - source: salt://sentry/sentry.conf.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - makedirs: True - -/etc/nginx/sites-enabled/sentry.conf: - file.symlink: - - target: /etc/nginx/sites-available/sentry.conf - - makedirs: True - - user: root - - group: root - - mode: 644 diff --git a/salt/salt/sentry/sentry-cron.service.jinja2 b/salt/salt/sentry/sentry-cron.service.jinja2 deleted file mode 100644 index c533422..0000000 --- a/salt/salt/sentry/sentry-cron.service.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Beat Service -After=network.target - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run cron - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry-web.service.jinja2 b/salt/salt/sentry/sentry-web.service.jinja2 deleted file mode 100644 index d140c28..0000000 --- a/salt/salt/sentry/sentry-web.service.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Main Service -After=network.target -Requires=sentry-worker.service -Requires=sentry-cron.service - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run web - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry-worker.service.jinja2 b/salt/salt/sentry/sentry-worker.service.jinja2 deleted file mode 100644 index 42d9373..0000000 --- a/salt/salt/sentry/sentry-worker.service.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% from 'sentry/common.jinja2' import sentry_bin_dir, sentry_cfg_dir, sentry_venv_dir -%} -[Unit] -Description=Sentry Background Worker -After=network.target - -[Service] -Type=simple -User=sentry -Group=sentry -WorkingDirectory={{ sentry_venv_dir }} -Environment=SENTRY_CONF={{ sentry_cfg_dir }} -ExecStart={{ sentry_bin_dir }}/sentry run worker - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/sentry/sentry.conf.jinja2 b/salt/salt/sentry/sentry.conf.jinja2 deleted file mode 100644 index 96ca7a5..0000000 --- a/salt/salt/sentry/sentry.conf.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - - {% for ip in pillar['developer_ips'] %} - allow {{ ip }}; - {% endfor %} - deny all; - - add_header Strict-Transport-Security "max-age={{ pillar['hsts_max_age'] }}; includeSubDomains; preload" always; - - server_name {{ pillar['sentry_server_name'] }}; - - location / { - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - proxy_pass http://localhost:9000; - } -} diff --git a/salt/salt/sentry/sentry.conf.py b/salt/salt/sentry/sentry.conf.py deleted file mode 100644 index 6e3459d..0000000 --- a/salt/salt/sentry/sentry.conf.py +++ /dev/null @@ -1,135 +0,0 @@ -# This file is just Python, with a touch of Django which means -# you can inherit and tweak settings to your hearts content. -from sentry.conf.server import * - -import os.path - -CONF_ROOT = os.path.dirname(__file__) - -DATABASES = { - 'default': { - 'ENGINE': 'sentry.db.postgres', - 'NAME': 'sentry', - 'USER': 'sentry', - 'PASSWORD': '', - 'HOST': '127.0.0.1', - 'PORT': '', - 'AUTOCOMMIT': True, - 'ATOMIC_REQUESTS': False, - } -} - -# You should not change this setting after your database has been created -# unless you have altered all schemas first -SENTRY_USE_BIG_INTS = True - -# If you're expecting any kind of real traffic on Sentry, we highly recommend -# configuring the CACHES and Redis settings - -########### -# General # -########### - -# Instruct Sentry that this install intends to be run by a single organization -# and thus various UI optimizations should be enabled. -SENTRY_SINGLE_ORGANIZATION = True -DEBUG = False - -######### -# Cache # -######### - -# Sentry currently utilizes two separate mechanisms. While CACHES is not a -# requirement, it will optimize several high throughput patterns. - -# If you wish to use memcached, install the dependencies and adjust the config -# as shown: -# -# pip install python-memcached -# -# CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', -# 'LOCATION': ['127.0.0.1:11211'], -# } -# } - -# A primary cache is required for things such as processing events -SENTRY_CACHE = 'sentry.cache.redis.RedisCache' - -######### -# Queue # -######### - -# See https://docs.sentry.io/on-premise/server/queue/ for more -# information on configuring your queue broker and workers. Sentry relies -# on a Python framework called Celery to manage queues. - -BROKER_URL = 'redis://localhost:6379' - -############### -# Rate Limits # -############### - -# Rate limits apply to notification handlers and are enforced per-project -# automatically. - -SENTRY_RATELIMITER = 'sentry.ratelimits.redis.RedisRateLimiter' - -################## -# Update Buffers # -################## - -# Buffers (combined with queueing) act as an intermediate layer between the -# database and the storage API. They will greatly improve efficiency on large -# numbers of the same events being sent to the API in a short amount of time. -# (read: if you send any kind of real data to Sentry, you should enable buffers) - -SENTRY_BUFFER = 'sentry.buffer.redis.RedisBuffer' - -########## -# Quotas # -########## - -# Quotas allow you to rate limit individual projects or the Sentry install as -# a whole. - -SENTRY_QUOTAS = 'sentry.quotas.redis.RedisQuota' - -######## -# TSDB # -######## - -# The TSDB is used for building charts as well as making things like per-rate -# alerts possible. - -SENTRY_TSDB = 'sentry.tsdb.redis.RedisTSDB' - -########### -# Digests # -########### - -# The digest backend powers notification summaries. - -SENTRY_DIGESTS = 'sentry.digests.backends.redis.RedisBackend' - -############## -# Web Server # -############## - -# If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto -# header and uncomment the following settings -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -SESSION_COOKIE_SECURE = True -CSRF_COOKIE_SECURE = True - -# If you're not hosting at the root of your web server, -# you need to uncomment and set it to the path where Sentry is hosted. -#FORCE_SCRIPT_NAME = '/sentry' - -SENTRY_WEB_HOST = '0.0.0.0' -SENTRY_WEB_PORT = 9000 -SENTRY_WEB_OPTIONS = { - # 'workers': 3, # the number of web workers - # 'protocol': 'uwsgi', # Enable uwsgi protocol instead of http -} diff --git a/salt/salt/tildes-wiki.sls b/salt/salt/tildes-wiki.sls deleted file mode 100644 index a6c2478..0000000 --- a/salt/salt/tildes-wiki.sls +++ /dev/null @@ -1,23 +0,0 @@ -{% from 'common.jinja2' import app_username %} - -# Create the base directory for the wiki files and initialize a git repo -/var/lib/tildes-wiki: - file.directory: - - user: {{ app_username }} - - group: {{ app_username }} - - mode: 775 - git.present: - - bare: False - - user: {{ app_username }} - -# Create the initial (empty) commit in the new git repo -wiki-initial-commit: - cmd.run: - - names: - - git config user.name "Tildes" - - git config user.email "Tildes" - - git commit --allow-empty -m "Initial commit" - - cwd: /var/lib/tildes-wiki - - runas: {{ app_username }} - - onchanges: - - file: /var/lib/tildes-wiki diff --git a/salt/salt/top.sls b/salt/salt/top.sls deleted file mode 100644 index 9ef97a7..0000000 --- a/salt/salt/top.sls +++ /dev/null @@ -1,46 +0,0 @@ -base: - 'dev or prod': - - general-config - - gunicorn - - nginx - - nginx.site-config - - postgresql - - postgresql.site-db - - postgresql.pgbouncer - - python - - redis - - redis.modules.redis-cell - - postgresql-redis-bridge - - scripts - - cmark-gfm - - prometheus.exporters.node_exporter - - prometheus.exporters.postgres_exporter - - prometheus.exporters.redis_exporter - - consumers - - tildes-wiki - - boussole - - webassets - - pts-lbsearch - - cronjobs - - final-setup # keep this state file last - 'dev': - - postgresql.test-db - - self-signed-cert - - development - - prometheus - - nodejs - - java - 'prod': - - nginx.shortener-config - - nginx.static-sites-config - - postgresql.prod-config - - prod-config - 'monitoring': - - nginx - - self-signed-cert - - postgresql - - redis - - sentry - - grafana - - prometheus - - prometheus.exporters.blackbox_exporter diff --git a/salt/salt/webassets.sls b/salt/salt/webassets.sls deleted file mode 100644 index f2de67b..0000000 --- a/salt/salt/webassets.sls +++ /dev/null @@ -1,25 +0,0 @@ -{% from 'common.jinja2' import app_dir, app_username %} - -# webassets will crash the site unless this file exists, make sure it's always there -{{ app_dir }}/static/css/site-icons.css: - file.managed: - - user: {{ app_username }} - - group: {{ app_username }} - - create: True - - replace: False - -/etc/systemd/system/webassets.service: - file.managed: - - source: salt://webassets.service.jinja2 - - template: jinja - - user: root - - group: root - - mode: 644 - - require_in: - - service: webassets.service - -webassets.service: - service.running: - - enable: True - - require: - - pip: pip-installs