From 0261e875679f1bf63c8d689da7fc7e014597885d Mon Sep 17 00:00:00 2001
From: Stonewall Jackson <stonewall@sacredheartsc.com>
Date: Sat, 4 Feb 2023 01:23:43 -0500
Subject: initial commit

---
 playbooks/util/backup.yml           | 606 ++++++++++++++++++++++++++++++++++++
 playbooks/util/client_cert.yml      |  71 +++++
 playbooks/util/decomission_host.yml |  56 ++++
 playbooks/util/restore.yml          | 477 ++++++++++++++++++++++++++++
 playbooks/util/wireguard_config.yml |  49 +++
 5 files changed, 1259 insertions(+)
 create mode 100644 playbooks/util/backup.yml
 create mode 100644 playbooks/util/client_cert.yml
 create mode 100644 playbooks/util/decomission_host.yml
 create mode 100644 playbooks/util/restore.yml
 create mode 100644 playbooks/util/wireguard_config.yml

(limited to 'playbooks/util')

diff --git a/playbooks/util/backup.yml b/playbooks/util/backup.yml
new file mode 100644
index 0000000..0c99eea
--- /dev/null
+++ b/playbooks/util/backup.yml
@@ -0,0 +1,606 @@
+#################
+# Set backup name
+#################
+- hosts: localhost
+  tags: always
+  tasks:
+    - name: get current timestamp
+      setup:
+        filter: ansible_date_time
+
+    - name: create backup directory
+      file:
+        path: '{{ backup_path }}'
+        state: directory
+
+- hosts: all:localhost:!unmanaged
+  tags: always
+  tasks:
+    - name: set backup name
+      set_fact:
+        backup_name: '{{ backup_name | default(hostvars.localhost.ansible_date_time.iso8601_basic_short) }}'
+
+
+################
+# IMAP Mailboxes
+################
+- name: backup dovecot mailboxes
+  hosts: imap_servers
+  vars_files: ../../roles/dovecot/vars/main.yml
+  vars:
+    dovecot_backup_dir: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-mailboxes
+    dovecot_backup_tarball: '{{ dovecot_backup_dir }}.tar.gz'
+    dovecot_backup_sieve_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-sieve.tar.gz
+  tags: dovecot,imap
+  tasks:
+    - name: create backup directory
+      file:
+        path: '{{ dovecot_backup_dir }}'
+        owner: '{{ dovecot_vmail_user }}'
+        group: '{{ dovecot_vmail_user }}'
+        mode: 0770
+        state: directory
+
+    - name: collect dovecot users
+      command: doveadm user *
+      register: dovecot_users
+      changed_when: no
+
+    - name: export mailboxes
+      command: >-
+        doveadm -o plugin/quota= backup -n inbox -f -u {{ item | quote }}
+        mdbox:{{ dovecot_backup_dir | quote }}/{{ item | quote }}/mdbox:LAYOUT=fs
+      loop: '{{ dovecot_users.stdout_lines }}'
+
+    - name: compress backup directory
+      archive:
+        path: '{{ dovecot_backup_dir }}'
+        dest: '{{ dovecot_backup_tarball }}'
+        mode: 0400
+        remove: yes
+
+    - name: fetch mailbox tarball
+      fetch:
+        src: '{{ dovecot_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete mailbox tarball from remote host
+      file:
+        path: '{{ dovecot_backup_tarball }}'
+        state: absent
+
+    - name: compress sieve scripts
+      archive:
+        path:
+          - '{{ dovecot_vmail_dir }}/*/sieve'
+          - '{{ dovecot_vmail_dir }}/*/.dovecot.sieve'
+        dest: '{{ dovecot_backup_sieve_tarball }}'
+        mode: 0400
+
+    - name: fetch sieve tarball
+      fetch:
+        src: '{{ dovecot_backup_sieve_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete sieve tarball from remote host
+      file:
+        path: '{{ dovecot_backup_sieve_tarball }}'
+        state: absent
+
+
+##################
+# Rspamd Databases
+##################
+- name: backup rspamd databases
+  hosts: rspamd_servers
+  vars_files:
+    - ../../roles/redis/vars/main.yml
+    - ../../roles/rspamd/vars/main.yml
+  vars:
+    rspamd_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-rspamd.tar.gz
+  tags: rspamd
+  tasks:
+    - name: dump redis databases to disk
+      command:
+        cmd: redis-cli -p {{ item }}
+        stdin: save
+      loop:
+        - '{{ rspamd_redis_port }}'
+        - '{{ rspamd_redis_bayes_port }}'
+
+    - name: compress redis directory
+      archive:
+        path: '{{ redis_home }}'
+        dest: '{{ rspamd_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ rspamd_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ rspamd_backup_tarball }}'
+        state: absent
+
+
+###################
+# ZNC Configuration
+###################
+- name: backup znc configuration
+  hosts: znc_servers
+  vars_files: ../../roles/znc/vars/main.yml
+  vars:
+    znc_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-znc.tar.gz
+  tags: znc
+  tasks:
+    - name: compress znc directory
+      archive:
+        path: '{{ znc_home }}'
+        dest: '{{ znc_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ znc_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ znc_backup_tarball }}'
+        state: absent
+
+
+#########################
+# Syncthing Configuration
+#########################
+- name: backup syncthing configuration
+  hosts: syncthing_servers
+  vars_files: ../../roles/syncthing/vars/main.yml
+  vars:
+    syncthing_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-syncthing.tar.gz
+  tags: syncthing
+  tasks:
+    - name: compress syncthing directory
+      archive:
+        path: '{{ syncthing_home }}'
+        dest: '{{ syncthing_backup_tarball }}'
+        exclusion_patterns:
+          - '*/index-*.db*'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ syncthing_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ syncthing_backup_tarball }}'
+        state: absent
+
+
+##################
+# Git Repositories
+##################
+- name: backup git respositories
+  hosts: git_servers
+  vars_files: ../../roles/gitolite/vars/main.yml
+  vars:
+    git_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-git.tar.gz
+  tags: git
+  tasks:
+    - name: compress git directory
+      archive:
+        path: '{{ gitolite_home }}'
+        dest: '{{ git_backup_tarball }}'
+        exclusion_patterns:
+          - git/.ansible*
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ git_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ git_backup_tarball }}'
+        state: absent
+
+
+######################
+# PostgreSQL Databases
+######################
+- name: backup postgresql databases
+  hosts: postgresql_servers
+  vars_files: ../../roles/postgresql_server/vars/main.yml
+  vars:
+    postgresql_backup_file: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-pg_dumpall.sql
+    postgresql_backup_gzip: '{{ postgresql_backup_file }}.gz'
+  tags: postgres,postgresql
+  tasks:
+    - name: dump databases
+      command: pg_dumpall -f {{ postgresql_backup_file | quote }}
+      become: yes
+      become_user: '{{ postgresql_user }}'
+
+    - name: compress sql file
+      archive:
+        path: '{{ postgresql_backup_file }}'
+        dest: '{{ postgresql_backup_gzip }}'
+        mode: 0400
+        remove: yes
+
+    - name: fetch backup gzip
+      fetch:
+        src: '{{ postgresql_backup_gzip }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup gzip from remote
+      file:
+        path: '{{ postgresql_backup_gzip }}'
+        state: absent
+
+
+
+########################
+# Jellyfin Configuration
+########################
+- name: backup jellyfin configuration
+  hosts: jellyfin_servers
+  vars_files: ../../roles/jellyfin/vars/main.yml
+  vars:
+    jellyfin_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-jellyfin.tar.gz
+  tags: jellyfin
+  tasks:
+    - name: compress jellyfin directories
+      archive:
+        path:
+          - '{{ jellyfin_home }}/data'
+          - '{{ jellyfin_home }}/metadata'
+          - '{{ jellyfin_home }}/plugins'
+          - '{{ jellyfin_home }}/root'
+          - '{{ jellyfin_conf_dir }}'
+        dest: '{{ jellyfin_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ jellyfin_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ jellyfin_backup_tarball }}'
+        state: absent
+
+
+##################
+# Mediawiki Images
+##################
+- name: backup mediawiki images
+  hosts: wiki_servers
+  vars_files: ../../roles/mediawiki/vars/main.yml
+  vars:
+    mediawiki_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-mediawiki.tar.gz
+  tags: mediawiki,wiki
+  tasks:
+    - name: compress images directory
+      archive:
+        path: '{{ mediawiki_home }}/images'
+        dest: '{{ mediawiki_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ mediawiki_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ mediawiki_backup_tarball }}'
+        state: absent
+
+
+#########################
+# Photostructure Database
+#########################
+- name: backup photostructure database
+  hosts: photostructure_servers
+  vars_files: ../../roles/photostructure/vars/main.yml
+  vars:
+    photostructure_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-photostructure.tar
+  tags: photostructure
+  tasks:
+    - name: stop photostructure
+      systemd:
+        name: photostructure
+        state: stopped
+
+    - name: archive photostructure library
+      archive:
+        path: '{{ photostructure_library }}'
+        dest: '{{ photostructure_backup_tarball }}'
+        format: tar
+        mode: 0400
+
+    - name: start photostructure
+      systemd:
+        name: photostructure
+        state: started
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ photostructure_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+        validate_checksum: no  # The tarball is way too big.
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ photostructure_backup_tarball }}'
+        state: absent
+
+
+###############
+# Asterisk Data
+###############
+- name: backup asterisk data
+  hosts: asterisk_servers
+  vars_files: ../../roles/asterisk/vars/main.yml
+  vars:
+    asterisk_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-asterisk.tar.gz
+  tags: asterisk
+  tasks:
+    - name: stop asterisk
+      systemd:
+        name: asterisk
+        state: stopped
+
+    - name: compress asterisk directory
+      archive:
+        path: '{{ asterisk_data_dir }}'
+        dest: '{{ asterisk_backup_tarball }}'
+        mode: 0400
+
+    - name: start asterisk
+      systemd:
+        name: asterisk
+        state: started
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ asterisk_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ asterisk_backup_tarball }}'
+        state: absent
+
+
+####################
+# Cups Configuration
+####################
+- name: backup cups configuration
+  hosts: cups_servers
+  vars:
+    cups_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-cups.tar.gz
+  tags: cups
+  tasks:
+    - name: compress cups configuration
+      archive:
+        path:
+          - /etc/cups/ppd
+          - /etc/cups/printers.conf
+        dest: '{{ cups_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ cups_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ cups_backup_tarball }}'
+        state: absent
+
+
+####################
+# WebDAV Directories
+####################
+- name: backup webdav directories
+  hosts: dav_servers
+  vars_files: ../../roles/sabredav/vars/main.yml
+  vars:
+    sabredav_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-webdav.tar.gz
+  tags: dav,sabredav,webdav
+  tasks:
+    - name: compress webdav directory
+      archive:
+        path: '{{ sabredav_home }}/webdav'
+        dest: '{{ sabredav_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ sabredav_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ sabredav_backup_tarball }}'
+        state: absent
+
+
+###############
+# Hastebin Data
+###############
+- name: backup hastebin data
+  hosts: pastebin_servers
+  vars_files: ../../roles/hastebin/vars/main.yml
+  vars:
+    hastebin_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-hastebin.tar.gz
+  tags: pastebin,hastebin
+  tasks:
+    - name: compress paste directory
+      archive:
+        path: '{{ hastebin_data_dir }}'
+        dest: '{{ hastebin_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ hastebin_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ hastebin_backup_tarball }}'
+        state: absent
+
+
+##################
+# Psitransfer Data
+##################
+- name: backup psitransfer data
+  hosts: filedrop_servers
+  vars_files: ../../roles/psitransfer/vars/main.yml
+  vars:
+    psitransfer_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-psitransfer.tar.gz
+  tags: psitransfer
+  tasks:
+    - name: compress files directory
+      archive:
+        path: '{{ psitransfer_data_dir }}'
+        dest: '{{ psitransfer_backup_tarball }}'
+        mode: 0400
+
+    - name: fetch backup tarball
+      fetch:
+        src: '{{ psitransfer_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup tarball from remote host
+      file:
+        path: '{{ psitransfer_backup_tarball }}'
+        state: absent
+
+
+##################
+# Apache WWW files
+##################
+- name: backup public apache files
+  hosts: web_servers
+  vars_files:
+    - ../../roles/apache/vars/main.yml
+  vars:
+    apache_backup_tarball: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-www.tar.gz
+  tags: apache,www
+  tasks:
+    - when: apache_backup_dirs | default([]) | length > 0
+      block:
+        - name: compress www directory
+          archive:
+            path: "{{ apache_backup_dirs | map('regex_replace', '^', apache_public_dir~'/') }}"
+            dest: '{{ apache_backup_tarball }}'
+            mode: 0400
+
+        - name: fetch backup tarball
+          fetch:
+            src: '{{ apache_backup_tarball }}'
+            dest: '{{ backup_path }}/'
+            flat: yes
+
+        - name: delete backup tarball from remote host
+          file:
+            path: '{{ apache_backup_tarball }}'
+            state: absent
+
+
+####################
+# Unifi Controllers
+####################
+- name: backup unifi controllers
+  hosts: unifi_controllers
+  vars_files: ../../roles/unifi/vars/main.yml
+  tags: unifi
+  tasks:
+    - name: collect autobackup files
+      find:
+        paths: '{{ unifi_autobackup_dir }}'
+        patterns: '*.unf'
+        file_type: file
+      register: unifi_autobackups
+
+    - name: fetch most recent autobackup file
+      fetch:
+        src: "{{ unifi_autobackups.files | sort(attribute='mtime') | map(attribute='path') | last }}"
+        dest: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-unifi.unf'
+        flat: yes
+
+
+################
+# FreeIPA Domain
+################
+- name: backup freeipa domain
+  hosts: freeipa_master
+  vars_files: ../../roles/freeipa_server/vars/main.yml
+  vars:
+    freeipa_backup_tarball: /var/tmp/{{ backup_name }}-ipa-{{ freeipa_realm }}.tar.gz
+  tags: ipa,freeipa
+  tasks:
+    - name: create full ipa backup
+      command: ipa-backup
+
+    - name: collect files in backup directory
+      find:
+        paths: '{{ freeipa_backup_dir }}'
+        patterns: ipa-full-*
+        file_type: directory
+      register: freeipa_backups
+
+    - name: compress latest backup
+      archive:
+        path: "{{ freeipa_backups.files | sort(attribute='mtime') | map(attribute='path') | last }}"
+        dest: '{{ freeipa_backup_tarball }}'
+        mode: 0400
+        remove: yes
+
+    - name: fetch backup archive
+      fetch:
+        src: '{{ freeipa_backup_tarball }}'
+        dest: '{{ backup_path }}/'
+        flat: yes
+
+    - name: delete backup archive from remote host
+      file:
+        path: '{{ freeipa_backup_tarball }}'
+        state: absent
+
+
+###############
+# Print summary
+###############
+- hosts: localhost
+  tags: always
+  tasks:
+    - debug:
+        msg: Backup {{ backup_name }} written to {{ backup_path }}.
diff --git a/playbooks/util/client_cert.yml b/playbooks/util/client_cert.yml
new file mode 100644
index 0000000..c81b298
--- /dev/null
+++ b/playbooks/util/client_cert.yml
@@ -0,0 +1,71 @@
+- name: generate client certificate
+  hosts: localhost
+  connection: local
+  become: no
+  vars_prompt:
+    - name: username
+      prompt: Enter username for the certificate subject
+      private: no
+    - name: passphrase
+      prompt: Enter password for the p12 file
+      private: yes
+  vars:
+    cert_dir: "{{ lookup('env', 'HOME') }}/pki"
+    key_size: 2048
+    key_path: '{{ cert_dir }}/{{ username }}.key'
+    csr_path: '{{ cert_dir }}/{{ username }}.csr'
+    crt_path: '{{ cert_dir }}/{{ username }}.crt'
+    p12_path: '{{ cert_dir }}/{{ username }}.p12'
+    profile_id: caIPAclientAuth
+  tasks:
+    - name: create output directory
+      file:
+        path: '{{ cert_dir }}'
+        state: directory
+
+    - name: generate private key
+      openssl_privatekey:
+        path: '{{ key_path }}'
+        size: '{{ key_size }}'
+        mode: 0600
+
+    - name: generate CSR
+      openssl_csr:
+        path: '{{ csr_path }}'
+        privatekey_path: '{{ key_path }}'
+        common_name: '{{ username }}'
+        use_common_name_for_san: no
+
+    - name: request certificate from IPA
+      shell:
+        cmd: >
+          ipa cert-request {{ csr_path }}
+          --principal {{ username }}
+          --profile-id {{ profile_id }}
+          --chain
+          --certificate-out {{ crt_path }}
+
+    # The openssl_pkcs12 ansible module seems to generate files that can't be
+    # decrypted by Android clients. The openssl CLI works fine though.
+    - name: generate PKCS#12 file
+      command:
+        cmd: >
+          openssl pkcs12 -export
+          -out {{ p12_path }}
+          -inkey {{ key_path }}
+          -in {{ crt_path }}
+          -name {{ username }}@{{ domain }}
+          -password pass:{{ passphrase | quote }}
+        creates: '{{ p12_path }}'
+
+    - name: cleanup files
+      file:
+        path: '{{ item }}'
+        state: absent
+      loop:
+        - '{{ key_path }}'
+        - '{{ csr_path }}'
+        - '{{ crt_path }}'
+
+    - debug:
+        msg: 'PKCS#12 file written to {{ p12_path }}. Passphrase: {{ passphrase }}'
diff --git a/playbooks/util/decomission_host.yml b/playbooks/util/decomission_host.yml
new file mode 100644
index 0000000..dae4b16
--- /dev/null
+++ b/playbooks/util/decomission_host.yml
@@ -0,0 +1,56 @@
+- name: decomission host
+  hosts: '{{ host }}'
+  tasks:
+    - name: delete A record
+      ipadnsrecord:
+        ipaadmin_principal: '{{ ipa_user }}'
+        ipaadmin_password: '{{ ipa_pass }}'
+        zone_name: '{{ domain }}'
+        record_name: '{{ host }}'
+        record_type: A
+        record_value: '{{ ip }}'
+        state: absent
+      delegate_to: '{{ freeipa_master }}'
+
+    - name: delete PTR record
+      ipadnsrecord:
+        ipaadmin_principal: '{{ ipa_user }}'
+        ipaadmin_password: '{{ ipa_pass }}'
+        zone_name: "{{ ip | ansible.utils.ipaddr('revdns') | regex_replace('^[^.]+\\.', '') }}"
+        record_name: '{{ ip.split(".") | last }}'
+        record_type: PTR
+        record_value: '{{ fqdn ~ "." }}'
+        state: absent
+      delegate_to: '{{ freeipa_master }}'
+
+    - name: delete CNAME records
+      ipadnsrecord:
+        ipaadmin_principal: '{{ ipa_user }}'
+        ipaadmin_password: '{{ ipa_pass }}'
+        zone_name: "{{ domain }}"
+        record_name: '{{ item.split(".") | first }}'
+        record_type: CNAME
+        record_value: '{{ fqdn ~ "." }}'
+        state: absent
+      delegate_to: '{{ freeipa_master }}'
+      loop: '{{ cnames }}'
+
+    - name: delete host object
+      ipahost:
+        ipaadmin_principal: '{{ ipa_user }}'
+        ipaadmin_password: '{{ ipa_pass }}'
+        name: '{{ fqdn }}'
+        state: absent
+      delegate_to: '{{ ipa_host }}'
+
+    - name: delete proxmox vm
+      proxmox_kvm:
+        node: '{{ proxmox_node }}'
+        api_host: localhost
+        api_user: '{{ proxmox_api_user }}'
+        api_password: '{{ proxmox_api_password }}'
+        name: '{{ inventory_hostname }}'
+        force: yes
+        state: absent
+      delegate_to: '{{ proxmox_api_host }}'
+      when: "'proxmox_instances' in group_names"
diff --git a/playbooks/util/restore.yml b/playbooks/util/restore.yml
new file mode 100644
index 0000000..3a0154c
--- /dev/null
+++ b/playbooks/util/restore.yml
@@ -0,0 +1,477 @@
+################
+# IMAP Mailboxes
+################
+- name: restore dovecot mailboxes
+  hosts: imap_servers
+  vars_files: ../../roles/dovecot/vars/main.yml
+  vars:
+    dovecot_temp_dir: /var/tmp/{{ backup_name }}-{{ inventory_hostname }}-mailboxes
+    dovecot_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-mailboxes.tar.gz'
+    dovecot_backup_sieve_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-sieve.tar.gz'
+  tags: dovecot,imap
+  tasks:
+    - name: create temporary directory
+      file:
+        path: '{{ dovecot_temp_dir }}'
+        owner: '{{ dovecot_vmail_user }}'
+        group: '{{ dovecot_vmail_user }}'
+        mode: 0770
+        state: directory
+
+    - name: extract mailbox tarball
+      unarchive:
+        src: '{{ dovecot_backup_tarball }}'
+        dest: '{{ dovecot_temp_dir }}'
+        extra_opts:
+          - --same-owner
+          - --strip-components=1
+
+    - name: collect dovecot users
+      command: doveadm user *
+      register: dovecot_users
+      changed_when: no
+
+    - name: import mailboxes
+      command: >-
+        doveadm -o plugin/quota= sync -u {{ item | quote }}
+        mdbox:{{ dovecot_temp_dir }}/{{ item | quote }}/mdbox
+      loop: '{{ dovecot_users.stdout_lines }}'
+
+    - name: drop FTS indexes
+      command: doveadm fts rescan -A
+
+    - name: reindex mailboxes
+      command: doveadm index -A -q *
+
+    - name: delete temporary directory
+      file:
+        path: '{{ dovecot_temp_dir }}'
+        state: absent
+
+    - name: extract sieve scripts
+      unarchive:
+        src: '{{ dovecot_backup_sieve_tarball }}'
+        dest: '{{ dovecot_vmail_dir }}'
+        extra_opts:
+          - --same-owner
+
+
+##################
+# Rspamd Databases
+##################
+- name: restore rspamd databases
+  hosts: rspamd_servers
+  vars_files:
+    - ../../roles/redis/vars/main.yml
+    - ../../roles/rspamd/vars/main.yml
+  vars:
+    rspamd_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-rspamd.tar.gz'
+  tags: rspamd
+  tasks:
+    - name: stop redis instances
+      systemd:
+        name: redis@{{ item }}
+        state: stopped
+      loop:
+        - '{{ rspamd_redis_port }}'
+        - '{{ rspamd_redis_bayes_port }}'
+
+    - name: stop rspamd
+      systemd:
+        name: rspamd
+        state: stopped
+
+    - name: extract redis tarballs
+      unarchive:
+        src: '{{ rspamd_backup_tarball }}'
+        dest: '{{ redis_home }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: start redis instances
+      systemd:
+        name: redis@{{ item }}
+        state: started
+      loop:
+        - '{{ rspamd_redis_port }}'
+        - '{{ rspamd_redis_bayes_port }}'
+
+    - name: start rspamd
+      systemd:
+        name: rspamd
+        state: started
+
+
+###################
+# ZNC Configuration
+###################
+- name: restore znc configuration
+  hosts: znc_servers
+  vars_files: ../../roles/znc/vars/main.yml
+  vars:
+    znc_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-znc.tar.gz'
+  tags: znc
+  tasks:
+    - name: stop znc
+      systemd:
+        name: znc
+        state: stopped
+
+    - name: extract config tarball
+      unarchive:
+        src: '{{ znc_backup_tarball }}'
+        dest: '{{ znc_home }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: start znc
+      systemd:
+        name: znc
+        state: started
+
+
+#########################
+# Syncthing Configuration
+#########################
+- name: restore syncthing configuration
+  hosts: syncthing_servers
+  vars_files: ../../roles/syncthing/vars/main.yml
+  vars:
+    syncthing_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-syncthing.tar.gz'
+  tags: syncthing
+  tasks:
+    - name: stop syncthing daemons
+      command: systemctl stop syncthing-user@*
+
+    - name: extract config tarball
+      unarchive:
+        src: '{{ syncthing_backup_tarball }}'
+        dest: '{{ syncthing_home }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: collect syncthing users
+      find:
+        paths: '{{ syncthing_home }}'
+        recurse: no
+        file_type: directory
+      register: syncthing_users
+
+    - name: start syncthing daemons
+      systemd:
+        name: syncthing-user@{{ item }}
+        state: started
+      loop: "{{ syncthing_users.files | map(attribute='path') | map('basename') }}"
+
+
+##################
+# Git Repositories
+##################
+- name: restore git repositories
+  hosts: git_servers
+  vars_files:
+    - ../../roles/gitolite/vars/main.yml
+    - ../../roles/cgit/vars/main.yml
+  vars:
+    git_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-git.tar.gz'
+  tags: git
+  tasks:
+    - name: extract git tarball
+      unarchive:
+        src: '{{ git_backup_tarball }}'
+        dest: '{{ gitolite_home }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: clear cgit cache
+      file:
+        path: '{{ cgit_cache_dir }}'
+        owner: apache
+        mode: 0755
+        setype: _default
+        state: '{{ item }}'
+      loop:
+        - absent
+        - directory
+
+
+######################
+# PostgreSQL Databases
+######################
+- name: restore postgresql databases
+  hosts: postgresql_servers
+  vars_files: ../../roles/postgresql_server/vars/main.yml
+  vars:
+    postgresql_backup_gzip: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-pg_dumpall.sql.gz'
+    postgresql_remote_gzip: /var/tmp/{{ postgresql_backup_gzip | basename }}
+  tags: postgres,postgresql
+  tasks:
+    - name: copy backup gzip to remote host
+      copy:
+        src: '{{ postgresql_backup_gzip }}'
+        dest: '{{ postgresql_remote_gzip }}'
+        owner: '{{ postgresql_user }}'
+        group: '{{ postgresql_user }}'
+        mode: 0400
+
+    - name: import database backup
+      shell: gunzip {{ postgresql_remote_gzip | quote }} --to-stdout | psql
+      become: yes
+      become_user: '{{ postgresql_user }}'
+
+    - name: delete gzip file from remote host
+      file:
+        path: '{{ postgresql_remote_gzip }}'
+        state: absent
+
+
+########################
+# Jellyfin Configuration
+########################
+- name: restore jellyfin configuration
+  hosts: jellyfin_servers
+  vars_files: ../../roles/jellyfin/vars/main.yml
+  vars:
+    jellyfin_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-jellyfin.tar.gz'
+  tags: jellyfin
+  tasks:
+    - name: stop jellyfin
+      systemd:
+        name: jellyfin
+        state: stopped
+
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ jellyfin_backup_tarball }}'
+        dest: /
+        extra_opts:
+          - --same-owner
+
+    - name: start jellyfin
+      systemd:
+        name: jellyfin
+        state: started
+
+
+##################
+# Mediawiki Images
+##################
+- name: restore mediawiki images
+  hosts: wiki_servers
+  vars_files: ../../roles/mediawiki/vars/main.yml
+  vars:
+    mediawiki_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-mediawiki.tar.gz'
+  tags: mediawiki,wiki
+  tasks:
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ mediawiki_backup_tarball }}'
+        dest: '{{ mediawiki_home }}/images'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+
+#########################
+# Photostructure Database
+#########################
+- name: restore photostructure database
+  hosts: photostructure_servers
+  vars_files:
+    - ../../roles/photostructure/vars/main.yml
+  vars:
+    photostructure_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-photostructure.tar'
+  tags: photostructure
+  tasks:
+    - name: stop photostructure
+      systemd:
+        name: photostructure
+        state: stopped
+
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ photostructure_backup_tarball }}'
+        dest: '{{ photostructure_library }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: start photostructure
+      systemd:
+        name: photostructure
+        state: started
+
+
+####################
+# Cups Configuration
+####################
+- name: restore cups configuration
+  hosts: cups_servers
+  vars:
+    cups_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-cups.tar.gz'
+  tags: cups
+  tasks:
+    - name: stop cups
+      systemd:
+        name: cups
+        state: stopped
+
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ cups_backup_tarball }}'
+        dest: /etc/cups
+        extra_opts:
+          - --same-owner
+
+    - name: start cups
+      systemd:
+        name: cups
+        state: started
+
+
+###############
+# Asterisk Data
+###############
+- name: restore asterisk data
+  hosts: asterisk_servers
+  vars_files: ../../roles/asterisk/vars/main.yml
+  vars:
+    asterisk_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-asterisk.tar.gz'
+  tags: asterisk
+  tasks:
+    - name: stop asterisk
+      systemd:
+        name: asterisk
+        state: stopped
+
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ asterisk_backup_tarball }}'
+        dest: '{{ asterisk_data_dir }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+    - name: start asterisk
+      systemd:
+        name: asterisk
+        state: started
+
+
+####################
+# WebDAV Directories
+####################
+- name: restore webdav directories
+  hosts: dav_servers
+  vars_files: ../../roles/sabredav/vars/main.yml
+  vars:
+    sabredav_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-webdav.tar.gz'
+  tags: sabredav,dav,webdav
+  tasks:
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ sabredav_backup_tarball }}'
+        dest: '{{ sabredav_home }}/webdav'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+
+###############
+# Hastebin Data
+###############
+- name: restore hastebin data
+  hosts: pastebin_servers
+  vars_files: ../../roles/hastebin/vars/main.yml
+  vars:
+    hastebin_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-hastebin.tar.gz'
+  tags: hastebin,pastebin
+  tasks:
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ hastebin_backup_tarball }}'
+        dest: '{{ hastebin_data_dir }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+
+##################
+# Psitransfer Data
+##################
+- name: restore psitransfer data
+  hosts: filedrop_servers
+  vars_files: ../../roles/psitransfer/vars/main.yml
+  vars:
+    psitransfer_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-psitransfer.tar.gz'
+  tags: psitransfer
+  tasks:
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ psitransfer_backup_tarball }}'
+        dest: '{{ psitransfer_data_dir }}'
+        extra_opts:
+          - --strip-components=1
+          - --same-owner
+
+
+##################
+# Apache WWW Files
+##################
+- name: restore public apache files
+  hosts: web_servers
+  vars_files: ../../roles/apache/vars/main.yml
+  vars:
+    apache_backup_tarball: '{{ backup_path }}/{{ backup_name }}-{{ inventory_hostname }}-www.tar.gz'
+  tags: apache,www
+  tasks:
+    - name: extract backup tarball
+      unarchive:
+        src: '{{ apache_backup_tarball }}'
+        dest: '{{ apache_public_dir }}'
+        extra_opts:
+          - --same-owner
+
+
+################
+# FreeIPA Domain
+################
+- name: restore freeipa domain
+  hosts: freeipa_master
+  vars_files: ../../roles/freeipa_server/vars/main.yml
+  vars:
+    freeipa_backup_tarball: '{{ backup_path }}/{{ backup_name }}-ipa-{{ freeipa_realm }}.tar.gz'
+    freeipa_remote_backup_path: '{{ freeipa_backup_dir }}/{{ backup_name }}'
+  tags: ipa,freeipa
+  tasks:
+    # Only restore FreeIPA when explicitly requested - it is quite disruptive.
+    - when: ansible_run_tags | intersect(['ipa','freeipa']) | length > 0
+      block:
+      - name: create backup directory on remote host
+        file:
+          path: '{{ freeipa_remote_backup_path }}'
+          state: directory
+          mode: 0700
+
+      - name: extract backup tarball
+        unarchive:
+          src: '{{ freeipa_backup_tarball }}'
+          dest: '{{ freeipa_remote_backup_path }}'
+          extra_opts:
+            - --strip-components=1
+            - --same-owner
+
+      - name: restore freeipa domain from backup
+        command: ipa-restore {{ backup_name | quote }} --unattended --password={{ freeipa_ds_password | quote }}
+
+      - name:  clear sssd cache
+        command: sss_cache -E
+
+      - name: delete backup files from remote host
+        file:
+          path: '{{ freeipa_remote_backup_path }}'
+          state: absent
diff --git a/playbooks/util/wireguard_config.yml b/playbooks/util/wireguard_config.yml
new file mode 100644
index 0000000..fb98ca4
--- /dev/null
+++ b/playbooks/util/wireguard_config.yml
@@ -0,0 +1,49 @@
+- name: generate client certificate
+  hosts: localhost
+  connection: local
+  become: no
+  vars_prompt:
+    - name: client_ip
+      prompt: Enter client ip address
+      private: no
+  vars:
+    config_path: "{{ lookup('env', 'HOME') }}/{{ organization | replace(' ', '-') | lower }}-wg.conf"
+    server_pubkey: '{{ wireguard_pubkey }}'
+    server_port: '{{ wireguard_port | default(51820) }}'
+    server_host: '{{ wireguard_host }}'
+    gateway: '{{ vlans.vpn.gateway }}'
+    dns_server: "{{ vlans.vpn.dns_servers | join(',') }}"
+  tasks:
+    - name: generate private key
+      command:
+        cmd: wg genkey
+      register: wg_genkey
+      changed_when: no
+
+    - name: generate public key
+      command:
+        cmd: wg pubkey
+        stdin: '{{ wg_genkey.stdout }}'
+      register: wg_pubkey
+      changed_when: no
+
+    - name: generate wireguard config file
+      copy:
+        dest: '{{ config_path }}'
+        mode: 0600
+        content: |
+          [Interface]
+          Address = {{ client_ip }}/32
+          PrivateKey = {{ wg_genkey.stdout }}
+          DNS = {{ dns_server }}
+
+          [Peer]
+          PublicKey = {{ server_pubkey }}
+          AllowedIPs = 0.0.0.0/0
+          Endpoint = {{ server_host }}:{{ server_port }}
+
+    - debug:
+        msg: 'wireguard client config written to {{ config_path }}'
+
+    - debug:
+        msg: 'Add the following client to the wireguard server: {{ client_ip }}/32 {{ wg_pubkey.stdout }}'
-- 
cgit