Serve randomized decoy sites with robots.txt (#2)

* Serve randomized decoy sites with robots.txt for masked proxies

* Run Debian host provisioning on PRs and pushes

* Pass CI SSH identity and known hosts to ProxyJump

* Diagnose two-host SSH jump authentication failures

* Reload SSH after successful configuration validation

Author: andre487

.github/workflows/integration.yml

@@ -44,7 +44,6 @@ jobs:
 
   two-host-jump:
     name: Debian 12 two-host Jump
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     timeout-minutes: 35
     steps:

docs/en/README.md

@@ -149,6 +149,11 @@ that wait for `407` before sending credentials. Keep it disabled unless tested w
 clients. A route can override the entry setting; `knock` allows selected hostnames to receive the
 normal challenge.
 
+Masked routes serve a static site using GOST's built-in loopback file server. A standard-library
+Python script generates a random heading and abstract SVG once. Repeated `apply` runs keep the
+site; remove `/opt/megaproxy/decoy/index.html` to regenerate it on the next `apply`.
+The page includes `noindex,nofollow`; `/robots.txt` contains `User-agent: *` and `Disallow: /`.
+
 ## HTTPS certificates and chains
 
 Domain certificates are issued with Certbot standalone ACME. An entry certificate contains its

docs/ru/README.md

@@ -155,6 +155,12 @@ Probe resistance подавляет обычный запрос аутентиф
 нужных клиентов. Маршрут может переопределить настройку entry; `knock` разрешает обычный запрос
 аутентификации для выбранных hostname-ов.
 
+При включённом маскировании GOST отдаёт статический сайт через встроенный файловый сервер
+на loopback. Python без дополнительных библиотек один раз создаёт случайный заголовок и
+абстрактную SVG-картинку. Повторный `apply` сохраняет сайт; удаление
+`/opt/megaproxy/decoy/index.html` приводит к новой генерации при следующем `apply`.
+Страница содержит `noindex,nofollow`, а `/robots.txt` — `User-agent: *` и `Disallow: /`.
+
 ## HTTPS-сертификаты и chains
 
 Доменные сертификаты выпускаются Certbot в standalone-режиме. Сертификат entry-сервера содержит его

playbooks/verify.yml

@@ -87,17 +87,16 @@
             ['curl', '--silent', '--show-error', '--fail', '--max-time', '15',
              '--write-out', '\n%{http_code}']
             + (['--insecure'] if megaproxy_services.https.certificate == 'self-signed' else [])
-            + ['https://' ~ item.hostname ~ ':' ~ (item.port | string) ~ '/']
+            + ['https://' ~ item.hostname ~ ':' ~ (megaproxy_services.https.port | string) ~ '/']
           }}
       register: decoy_response
       changed_when: false
       delegate_to: localhost
       become: false
-      loop: "{{ megaproxy_https_public_routes | default([]) }}"
+      loop: "{{ megaproxy_https_public_routes | default([]) | selectattr('probe_resistance_enabled') | list }}"
       when:
         - megaproxy_services.https is defined
         - megaproxy_services.https.enabled | bool
-        - megaproxy_services.https.probe_resistance.enabled | bool
 
     - name: Ensure decoy does not disclose proxy software
       ansible.builtin.assert:
@@ -107,9 +106,33 @@
           - "'proxy-authenticate' not in (item.stdout | lower)"
           - "'gost' not in (item.stdout | lower)"
           - "'megaproxy' not in (item.stdout | lower)"
+          - "'content=\"noindex,nofollow\"' in item.stdout"
       loop: "{{ decoy_response.results | default([]) }}"
       when: megaproxy_services.https is defined and megaproxy_services.https.enabled | bool
 
+    - name: Read robots.txt from masked routes
+      ansible.builtin.command:
+        argv: >-
+          {{
+            ['curl', '--silent', '--show-error', '--fail', '--max-time', '15']
+            + (['--insecure'] if megaproxy_services.https.certificate == 'self-signed' else [])
+            + ['https://' ~ item.hostname ~ ':' ~ (megaproxy_services.https.port | string) ~ '/robots.txt']
+          }}
+      loop: "{{ megaproxy_https_public_routes | default([]) | selectattr('probe_resistance_enabled') | list }}"
+      register: robots_responses
+      changed_when: false
+      delegate_to: localhost
+      become: false
+      when: megaproxy_services.https is defined and megaproxy_services.https.enabled | bool
+
+    - name: Require robots.txt to disallow crawling
+      ansible.builtin.assert:
+        that:
+          - >-
+            item.stdout == 'User-agent: *\nDisallow: /'
+      loop: "{{ robots_responses.results | default([]) }}"
+      when: item is not skipped
+
     - name: Probe CONNECT without credentials
       ansible.builtin.command:
         argv:

roles/admin/handlers/main.yml

@@ -1,7 +1,6 @@
 ---
 - name: Validate and reload administrative ssh policy
   ansible.builtin.command: /usr/sbin/sshd -t
-  changed_when: false
   notify: Reload ssh after administrative policy
 
 - name: Reload ssh after administrative policy

roles/https_proxy/files/generate-decoy.py

@@ -0,0 +1,47 @@
+"""Generate a small static site using only the Python standard library."""
+
+import random
+import sys
+from pathlib import Path
+
+
+def generate(directory):
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    title = f"{random.choice(('Quiet', 'Soft', 'Distant', 'Golden', 'Hidden', 'Open'))} {random.choice(('Horizons', 'Shapes', 'Reflections', 'Gardens', 'Waves', 'Spaces'))}"
+    hue = random.randrange(360)
+    shapes = []
+    for _ in range(18):
+        x, y, radius = random.randrange(800), random.randrange(480), random.randrange(30, 180)
+        color = f"hsl({(hue + random.randrange(90)) % 360},55%,65%)"
+        shapes.append(f'<circle cx="{x}" cy="{y}" r="{radius}" fill="{color}" opacity="0.45"/>')
+    image = (
+        '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 480">'
+        f'<rect width="800" height="480" fill="hsl({hue},25%,94%)"/>'
+        + ''.join(shapes) + '</svg>\n'
+    )
+    (directory / 'art.svg').write_text(image, encoding='utf-8')
+    (directory / 'robots.txt').write_text('User-agent: *\nDisallow: /\n', encoding='utf-8')
+    html = f'''<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <meta name="robots" content="noindex,nofollow">
+  <title>{title}</title>
+  <style>body{{max-width:50rem;margin:8vh auto;padding:0 1.5rem;font:17px/1.6 system-ui,sans-serif;color:#28323c}}img{{width:100%;height:auto;border-radius:12px}}</style>
+</head>
+<body>
+  <main>
+    <h1>{title}</h1>
+    <p>A small study of colour, form, and light.</p>
+    <img src="/art.svg" alt="An abstract composition of overlapping colourful circles" width="800" height="480">
+  </main>
+</body>
+</html>
+'''
+    (directory / 'index.html').write_text(html, encoding='utf-8')
+
+
+if __name__ == '__main__':
+    generate(sys.argv[1])

roles/https_proxy/tasks/main.yml

@@ -96,14 +96,20 @@
     - megaproxy_services.https.certificate == "ip-acme"
     - not megaproxy_certificate.stat.exists
 
-- name: Install active-probe decoy page
-  ansible.builtin.template:
-    src: decoy.html.j2
-    dest: /opt/megaproxy/decoy.html
+- name: Create decoy directory
+  ansible.builtin.file:
+    path: /opt/megaproxy/decoy
+    state: directory
     owner: root
     group: root
-    mode: "0644"
-  notify: Restart MegaProxy GOST
+    mode: "0755"
+
+- name: Generate a random static decoy site
+  ansible.builtin.script:
+    cmd: generate-decoy.py /opt/megaproxy/decoy
+    executable: /usr/bin/python3
+    creates: /opt/megaproxy/decoy/index.html
+  when: megaproxy_services.https.routes | selectattr('probe_resistance.enabled') | list | length > 0
 
 - name: Install secret GOST configuration
   ansible.builtin.template:

roles/https_proxy/templates/decoy.html.j2

@@ -10,7 +10,7 @@ services:
 {% endif %}
       metadata:
 {% if route.probe_resistance.enabled %}
-        probeResist: "file:/opt/megaproxy/decoy.html"
+        probeResist: "host:127.0.0.1:18080"
 {% if route.probe_resistance.knock %}
         knock: {{ route.probe_resistance.knock | join(',') | to_json }}
 {% endif %}
@@ -28,6 +28,16 @@ services:
           maxVersion: VersionTLS13
           alpn: [h2, http/1.1]
 {% endfor %}
+{% if megaproxy_services.https.routes | selectattr('probe_resistance.enabled') | list %}
+  - name: decoy
+    addr: "127.0.0.1:18080"
+    handler:
+      type: file
+      metadata:
+        dir: /opt/megaproxy/decoy
+    listener:
+      type: tcp
+{% endif %}
 
 authers:
   - name: megaproxy-users

roles/https_proxy/templates/gost.yml.j2

@@ -7,7 +7,7 @@ Wants=network-online.target
 [Service]
 Type=simple
 ExecStartPre=-/usr/bin/docker rm -f megaproxy-gost
-ExecStart=/usr/bin/docker run --name megaproxy-gost --network host --read-only --cap-drop ALL --security-opt no-new-privileges --memory 256m --pids-limit 128 -v /opt/megaproxy/gost.yml:/etc/gost/gost.yml:ro -v /opt/megaproxy/decoy.html:/opt/megaproxy/decoy.html:ro -v {{ megaproxy_certificate_volume }} gogost/gost:{{ megaproxy_services.https.gost_version }} -C /etc/gost/gost.yml
+ExecStart=/usr/bin/docker run --name megaproxy-gost --network host --read-only --cap-drop ALL --security-opt no-new-privileges --memory 256m --pids-limit 128 -v /opt/megaproxy/gost.yml:/etc/gost/gost.yml:ro -v /opt/megaproxy/decoy:/opt/megaproxy/decoy:ro -v {{ megaproxy_certificate_volume }} gogost/gost:{{ megaproxy_services.https.gost_version }} -C /etc/gost/gost.yml
 ExecStop=/usr/bin/docker stop -t 10 megaproxy-gost
 Restart=on-failure
 RestartSec=5