haproxy, firewall, containers: force connections through bastion vpn

docker is now listening on localhost, with a haproxy on the services
server to forward the ports outwards. this is because docker tends to
disregard UFW's rules, but haproxy should be better in that regard.

meanwhile, the firewall rules have been configured properly to only
allow the bastion IP in over the wireguard connection, for proper
authentication.

inventory.example.yml

@@ -67,6 +67,7 @@ all:
     haproxy:
       hosts:
         your_bastion_host:
+        your_fleet_host:
     wireguard:
       hosts:
         your_bastion_host:

roles/containers/tasks/main.yml

@@ -126,10 +126,6 @@
   register: user_synapse
   when: '"synapse" in group_names'
 
-- name: Figure out local IP address
-  set_fact:
-    docker_ip: "{{ vpn_ip if wireguard_services else local_ip }}"
-
 - name: Generate docker-compose.yml
   template:
     src: "docker-compose.yml.j2"

roles/containers/templates/docker-compose.yml.j2

@@ -1,4 +1,9 @@
 # vim: ft=yaml
+
+
+# docker doesn't play well with the firewall, so i have it listen on 127.0.0.1
+# and have haproxy expose it publicly (won't disrespect firewall rules)
+
 ---
 
 networks:
@@ -20,8 +25,8 @@ services:
       - GITEA__server__DOMAIN={{ gitea_domain }}
       - GITEA__server__SSH_DOMAIN={{ gitea_domain }}
     ports:
-      - "{{ docker_ip }}:3000:3000"
-      - "{{ docker_ip }}:2498:22"
+      - "127.0.0.1:3000:3000"
+      - "127.0.0.1:2498:22"
     restart: unless-stopped
     volumes:
       - {{ dataroot }}/gitea:/data
@@ -63,7 +68,7 @@ services:
     networks:
       - navidrome
     ports:
-      - "{{ docker_ip }}:4533:4533"
+      - "127.0.0.1:4533:4533"
 
 {% endif %}
 {% if "synapse" in group_names %}
@@ -80,7 +85,7 @@ services:
     networks:
       - navidrome
     ports:
-      - "{{ docker_ip }}:8008:8008/tcp"
+      - "127.0.0.1:8008:8008/tcp"
 
 {% endif %}
 
@@ -99,7 +104,7 @@ services:
     depends_on:
       - paperless-broker
     ports:
-      - "{{ docker_ip }}:8000:8000"
+      - "127.0.0.1:8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

roles/firewall/tasks/main.yml

@@ -17,12 +17,18 @@
     default_firewall_src: "{{ bastion_vpn_ip if wireguard_services else bastion_ip }}"
   when: '"fleet" in group_names'
 
-# this is actually kind of useless because docker bypasses this
+- name: Configure service interface
+  set_fact:
+    service_firewall_if: "{{ wireguard_interface if wireguard_services else omit }}"
+  when: 'wireguard_services'
+
 - name: Allow service ports
   community.general.ufw:
     rule: allow
     port: "{{ item.port }}"
     proto: "{{ item.proto | default('tcp') }}"
+    # service -> VPN interface if available, else default
+    interface_in: "{{ service_firewall_if if (item.interface | default('')) == 'service' else item.interface | default(omit) }}"
     src: "{{ item.src | default(default_firewall_src) }}"
   when: item.name in group_names
   with_items:
@@ -31,10 +37,12 @@
       port: 8448
     - name: "synapse"
       port: 8008
+      interface: service
 
     # navidrome api/web interface
     - name: "navidrome"
       port: 4533
+      interface: service
 
     - name: "syncthing"
       port: 22000
@@ -48,9 +56,13 @@
     # gitea sshd
     - name: "bastion"
       port: 2499
+    - name: "gitea"
+      port: 2498
+      interface: service
     # gitea http
     - name: "gitea"
       port: 3000
+      interface: service
 
     - name: "caddy"
       port: 80
@@ -67,7 +79,7 @@
       proto: udp
       src: any
 
-- name: Deny all ports by default
+- name: Enable UFW
   community.general.ufw:
     state: enabled
 

roles/haproxy/tasks/main.yml

@@ -5,11 +5,41 @@
     name:
       - haproxy
 
-- name: Deploy haproxy config
+- name: Figure out local IP address
+  set_fact:
+    service_ip: "{{ vpn_ip if wireguard_services else local_ip }}"
+
+# for info about this, see top of roles/containers/templates/docker-compose.yml.j2
+- name: Enumerate services to forward
+  set_fact:
+    haproxy_services:
+      - name: gitea
+        ports:
+          - 2498
+          - 3000
+      - name: navidrome
+        ports:
+          - 4533
+      - name: synapse
+        ports:
+          - 8008
+      - name: paperless
+        ports:
+          - 8000
+
+- name: Deploy haproxy config (bastion)
   template:
-    src: haproxy.cfg.j2
+    src: haproxy.cfg.bastion.j2
     dest: /etc/haproxy/haproxy.cfg
     lstrip_blocks: true
+  when: '"bastion" in group_names'
+
+- name: Deploy haproxy config (fleet)
+  template:
+    src: haproxy.cfg.fleet.j2
+    dest: /etc/haproxy/haproxy.cfg
+    lstrip_blocks: true
+  when: '"fleet" in group_names'
 
 - name: Enable haproxy service
   systemd:

roles/haproxy/templates/haproxy.cfg.bastion.j2

@@ -15,12 +15,3 @@ listen gitea_ssh
 	server {{ host }} {{ host }}:2498
 {% endfor %}
 {% endif %}
-
-
-{% if groups["syncthing"] | length > 0 and "syncthing" not in group_names %}
-listen syncthing
-	bind *:22000
-{% for host in groups["syncthing"] %}
-	server {{ host }} {{ host }}:22000
-{% endfor %}
-{% endif %}

roles/haproxy/templates/haproxy.cfg.fleet.j2

@@ -0,0 +1,17 @@
+defaults
+	log global
+	mode tcp
+	timeout connect 10s
+	timeout client 36h
+	timeout server 36h
+	balance leastconn
+
+{% for service in haproxy_services %}
+{% if service.name in group_names %}
+{% for port in service.ports %}
+listen {{ service.name}}{{ port }}
+	bind {{ service_ip }}:{{ port }}
+	server localhost 127.0.0.1:{{ port }}
+{% endfor %}
+{% endif %}
+{% endfor %}

roles/synapse/templates/homeserver.yaml.j2

@@ -1,4 +1,4 @@
-# https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
+# https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html
 
 server_name: "{{ matrix_domain }}"
 pid_file: /data/homeserver.pid
@@ -12,6 +12,8 @@ listeners:
       - names: [client, federation]
         compress: false
 
+max_upload_size: 500M
+
 database:
   name: sqlite3
   args: