From f82b44aebb7048af93617305487961e7659f112c Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 20 Oct 2026 01:41:33 -0700 Subject: [PATCH] Enable QUIC address discovery on both Iroh relays Share the relay runtime across Fedora and Snow Globe, with certificate-backed UDP discovery alongside Caddy HTTPS. Use private systemd credentials and Certbot renewal on r1, and a DNS certificate sidecar with live certificate reload on r2. Keep previews on dynamic UDP ports and extend the persistent public-port mappings for production. Verified Iroh 1.3 relay-only transfers on both hosts, trusted-TLS UDP address discovery on r1 and the isolated r2 preview, r1 certificate-renewal rehearsal, Nix firewall/NAT evaluation, and seven deployment tests. Assisted-by: gpt-6-astra --- nixos/snowglobe.nix | 9 ++-- service/iroh-relay/Dockerfile | 7 ++- service/iroh-relay/iroh-relay.service | 5 +- service/iroh-relay/network.json | 1 + service/iroh-relay/r1.caddy | 8 +++- service/iroh-relay/relay.toml | 2 - service/iroh-relay/run.py | 50 ++++++++++++++++++++ service/iroh-relay/service.pkl | 58 ++++++++++++++++++----- tools/iroh-relay.md | 68 +++++++++++++++++++++++++++ tools/xmpp.md | 2 +- 10 files changed, 188 insertions(+), 22 deletions(-) create mode 100644 service/iroh-relay/network.json delete mode 100644 service/iroh-relay/relay.toml create mode 100644 service/iroh-relay/run.py create mode 100644 tools/iroh-relay.md diff --git a/nixos/snowglobe.nix b/nixos/snowglobe.nix index 8e7a4ab384309981f4450d479d2f33c103de9de4..0ff4136f4a57d86cf58dbe0e7ad84ae54307a514 100644 --- a/nixos/snowglobe.nix +++ b/nixos/snowglobe.nix @@ -5,6 +5,7 @@ let xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json); xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; + irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); encryptionRoots = { apps = "apps"; clover = "clover"; @@ -24,7 +25,7 @@ in networking.hostId = "4fa19ccb"; services.nomad.settings.name = config.networking.hostName; networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; - networking.firewall.allowedUDPPorts = xmppUDPPorts; + networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ]; networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept @@ -34,7 +35,7 @@ in ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept ''; - systemd.services.xmpp-nat = { + systemd.services.public-ports = { wantedBy = [ "multi-user.target" ]; wants = [ "network-online.target" ]; after = [ "network-online.target" ]; @@ -42,9 +43,9 @@ in serviceConfig = { Type = "oneshot"; DynamicUser = true; }; script = '' description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" - exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe XMPP" -r \ + exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ - ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} + ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} ''; }; diff --git a/service/iroh-relay/Dockerfile b/service/iroh-relay/Dockerfile index dcdd9ccf639b329a04f813e3a2688a368db32827..a6151e1a7e97e27df1aee7b5e058e73fa447101f 100644 --- a/service/iroh-relay/Dockerfile +++ b/service/iroh-relay/Dockerfile @@ -1,6 +1,9 @@ -FROM docker.io/library/alpine:3.22@sha256:3e9b4b680bfc9fb5269227cffbd6d42be39fbf7c0b908123913864aa4447e764 +FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332 +RUN apt-get update && apt-get install -y --no-install-recommends certbot python3-certbot-dns-cloudflare \ + && rm -rf /var/lib/apt/lists/* ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \ && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \ && rm /tmp/iroh-relay.tar.gz -ENTRYPOINT ["/usr/local/bin/iroh-relay"] +COPY run.py network.json /app/ +ENTRYPOINT ["python3", "/app/run.py"] diff --git a/service/iroh-relay/iroh-relay.service b/service/iroh-relay/iroh-relay.service index 011bb3126a4e6437fefd51cad97b8a560f7cde20..d16095a145268f543996fbae48ac580f6e195ba8 100644 --- a/service/iroh-relay/iroh-relay.service +++ b/service/iroh-relay/iroh-relay.service @@ -5,7 +5,10 @@ After=network-online.target [Service] DynamicUser=true -ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml +Environment=HTTP_PORT=3340 +LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem +LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem +ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay Restart=on-failure RestartSec=5 NoNewPrivileges=true diff --git a/service/iroh-relay/network.json b/service/iroh-relay/network.json new file mode 100644 index 0000000000000000000000000000000000000000..96598cf205d0c7e52f643eecf00605dd01952dc0 --- /dev/null +++ b/service/iroh-relay/network.json @@ -0,0 +1 @@ +{"quic":7842} diff --git a/service/iroh-relay/r1.caddy b/service/iroh-relay/r1.caddy index a1bb67898e18281db7251f49d678b34a4e24934a..1cd75f158c378d64d4f666a7d2e3b7d90f84f511 100644 --- a/service/iroh-relay/r1.caddy +++ b/service/iroh-relay/r1.caddy @@ -1,3 +1,9 @@ r1.iroh.paperclover.net { - reverse_proxy 127.0.0.1:3340 + handle /.well-known/acme-challenge/* { + root * /var/lib/iroh-acme + file_server + } + handle { + reverse_proxy 127.0.0.1:3340 + } } diff --git a/service/iroh-relay/relay.toml b/service/iroh-relay/relay.toml deleted file mode 100644 index 41936fdbf936cba857e791a2581a536475e14044..0000000000000000000000000000000000000000 --- a/service/iroh-relay/relay.toml +++ /dev/null @@ -1,2 +0,0 @@ -http_bind_addr = "0.0.0.0:3340" -enable_metrics = false diff --git a/service/iroh-relay/run.py b/service/iroh-relay/run.py new file mode 100644 index 0000000000000000000000000000000000000000..6140475cd0eb5d3fe6ff2d460ba34f3f5dffea8b --- /dev/null +++ b/service/iroh-relay/run.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import subprocess +import sys +import time + + +os.umask(0o077) +mode = sys.argv[1] +tls = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/tls/live/server")) + +if mode == "certificates": + credentials = Path("/tmp/cloudflare.ini") + credentials.write_text("dns_cloudflare_api_token = " + os.environ.pop("DNS_TOKEN") + "\n") + command = [ + "certbot", "certonly", "--non-interactive", "--agree-tos", + "--email", os.environ["OWNER_EMAIL"], "--dns-cloudflare", + "--dns-cloudflare-credentials", str(credentials), + "--dns-cloudflare-propagation-seconds", "30", "--cert-name", "server", + "--config-dir", "/tls", "--work-dir", "/tmp/certbot-work", "--logs-dir", "/tmp/certbot-logs", + "--keep-until-expiring", "-d", os.environ["RELAY_HOST"], + ] + while True: + subprocess.run(command, check=not (tls / "fullchain.pem").exists(), timeout=300) + time.sleep(12 * 60 * 60) + +config = Path("/tmp/iroh-relay.toml") +if mode == "relay": + for _ in range(180): + if (tls / "fullchain.pem").exists() and (tls / "privkey.pem").exists(): + break + time.sleep(1) + else: + raise RuntimeError("Relay certificate was not issued within three minutes") + quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( + Path(__file__).with_name("network.json").read_text())["quic"] + config.write_text( + 'enable_metrics = false\nenable_quic_addr_discovery = true\n' + f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' + '[tls]\ncert_mode = "Reloading"\n' + f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' + f'manual_cert_path = {json.dumps(str(tls / "fullchain.pem"))}\n' + f'manual_key_path = {json.dumps(str(tls / "privkey.pem"))}\n' + ) +else: + raise ValueError("Unknown relay task") +# --dev leaves HTTPS to Caddy while retaining certificate-backed QUIC discovery. +os.execv("/usr/local/bin/iroh-relay", ["iroh-relay", "--dev", "--config-path", str(config)]) diff --git a/service/iroh-relay/service.pkl b/service/iroh-relay/service.pkl index 183fe86b67857f5e03d464fd77e1dcf4b86e474b..b05394567ad69549a0c768a1a836974cf6517196 100644 --- a/service/iroh-relay/service.pkl +++ b/service/iroh-relay/service.pkl @@ -1,19 +1,55 @@ amends "../../config/Service.pkl" +import "../../config/site.pkl" as site +import "../ddns-updater/service.pkl" as dns +import "pkl:json" + +local network = new json.Parser {}.parse(read("network.json").text) +local host = read?("prop:hostname") ?? "r2.iroh.\(site.domain)" + name = "Iroh relay" launcher = false stageIsolation = "fresh" +requirements { new dns.DnsCredentials {} } -container { - build = "." - readOnlyRoot = true - http { - containerPort = 3340 - subdomain = "r2.iroh" - checkPath = "/healthz" +containers { + ["app"] { + build = "." + hostNetwork = true + readOnlyRoot = true + tmpfs { "/tmp" } + args { "relay" } + http { containerPort = 3340; hostname = host; loopback = true; checkPath = "/healthz" } + tcp { + new { + name = "quic" + containerPort = network.quic + hostPort = if (site.preview) null else network.quic + loopback = false + } + } + volumes { ["/tls"] { readOnly = true } } + envTemplate = """ + HTTP_PORT={{ env "NOMAD_PORT_app_http" }} + QUIC_PORT={{ env "NOMAD_PORT_app_quic" }} + """ + cpu = 200 + memory = 256 + } + ["certificates"] { + build = "." + hostNetwork = true + readOnlyRoot = true + tmpfs { "/tmp" } + lifecycle = "prestartSidecar" + args { "certificates" } + volumes { ["/tls"] {} } + env { + ["RELAY_HOST"] = host + ["OWNER_EMAIL"] = site.ownerEmail + ["DNS_TOKEN"] = "${secret.dns.api_token}" + } + cpu = 100 + memory = 256 } - args { "--config-path"; "/etc/iroh-relay.toml" } - volumes { ["/etc/iroh-relay.toml"] { config = "relay.toml" } } - cpu = 200 - memory = 256 } diff --git a/tools/iroh-relay.md b/tools/iroh-relay.md new file mode 100644 index 0000000000000000000000000000000000000000..e3124e0d479b84d1b6dfe0783704a07c04ceb67c --- /dev/null +++ b/tools/iroh-relay.md @@ -0,0 +1,68 @@ +# Iroh relays + +Clients use `https://r1.iroh.paperclover.net` and +`https://r2.iroh.paperclover.net`. Caddy terminates HTTPS on TCP 443; Iroh serves +QUIC address discovery on UDP 7842. TCP 80 remains available for ACME. +These are public relays without an application login. + +The upstream binary version and checksum live in +[`Dockerfile`](../service/iroh-relay/Dockerfile). Both machines use +[`run.py`](../service/iroh-relay/run.py) and +[`network.json`](../service/iroh-relay/network.json). +Iroh's `--dev` flag disables its HTTPS listener behind Caddy, while its +`Reloading` certificate resolver still supplies valid TLS for UDP discovery. + +## Snow Globe + +[`service.pkl`](../service/iroh-relay/service.pkl) declares the relay and its +certificate sidecar. The sidecar obtains the Cloudflare credential through the +DDNS provider, keeps it in a private tmpfs file, and checks certificate renewal +every twelve hours. Only the certificate volume is shared with the relay. +The relay reloads renewed certificates without restarting connections. + +Stage and deploy with the normal `tools/deploy.py` workflow. Fresh previews have +their own certificate storage and dynamically allocated UDP port; they do not +bind production's port. The host's `public-ports.service` maintains the production +router mapping; the firewall declaration is in `nixos/snowglobe.nix`. + +## paperclover.net + +Connect with `ssh root@paperclover.net`; the hostname is `usw-server-1`. +The checksum-verified upstream binary is installed at `/usr/local/bin/iroh-relay`. +Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and +[`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into +`/etc/systemd/system/`. + +[`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as +`/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main +Caddyfile. Validate before reloading Caddy. The ACME webroot is +`/var/lib/iroh-acme`; Caddy must be able to read it. + +After Caddy's HTTPS certificate is ready, issue the discovery certificate: + +```sh +certbot certonly --non-interactive --agree-tos --email account@paperclover.net \ + --webroot -w /var/lib/iroh-acme --cert-name iroh-relay \ + -d r1.iroh.paperclover.net \ + --deploy-hook 'systemctl try-restart iroh-relay.service' +systemctl daemon-reload +systemctl enable --now iroh-relay.service certbot-renew.timer +firewall-cmd --add-port=7842/udp +firewall-cmd --permanent --add-port=7842/udp +``` + +Systemd loads the certificate and key as private service credentials. Certbot's +renewal hook restarts the service to refresh those credentials. The key stays +root-only under `/etc/letsencrypt`. + +## Verification + +`GET /healthz` reports the relay version, but does not test packet forwarding. +Use two Iroh 1.3 endpoints with `clear_ip_transports()` and a custom relay map; +transfer and verify a payload in both directions. For UDP, connect a +`iroh_relay::quic::QuicClient` with normal CA validation and await its +`observed_external_addr()` stream. Test from outside the server's network. + +Renewal can be exercised on r1 with +`certbot renew --cert-name iroh-relay --dry-run --run-deploy-hooks`. +Inspect the relay's restart count and health afterward. diff --git a/tools/xmpp.md b/tools/xmpp.md index 5841fd7fedc6a719e50d2c98697590fa1f618840..3414536acd0b44a7402ed6138617a5c352c4774f 100644 --- a/tools/xmpp.md +++ b/tools/xmpp.md @@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide provisions client, direct-TLS client, and federation SRV records pointing at `xmpp.paperclover.net`; the website keeps its existing apex address. Router forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range -to Snow Globe. `xmpp-nat.service` refreshes UPnP mappings on startup and every five +to Snow Globe. `public-ports.service` refreshes UPnP mappings on startup and every five minutes, discovering the current description URL with a bounded unicast SSDP request to the configured gateway on port 1900. Replies and HTTP description hosts must match that gateway; discovery failure fails the refresh. Multicast -- 2.54.1