| author | |
| committer | |
| log | f82b44aebb7048af93617305487961e7659f112c |
| tree | e5f17048898f7c93453dc1c5a93eca69a1547a10 |
| parent | a235692734ea58ecdc64a0afc94800d595be91e2 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Share the relay runtime across Fedora and Snow Globe, with certificate-backed UDP discovery alongside Caddy HTTPS. Use private systemd credentials and Certbot renewal on r1, and a DNS certificate sidecar with live certificate reload on r2. Keep previews on dynamic UDP ports and extend the persistent public-port mappings for production.
Verified Iroh 1.3 relay-only transfers on both hosts, trusted-TLS UDP address discovery on r1 and the isolated r2 preview, r1 certificate-renewal rehearsal, Nix firewall/NAT evaluation, and seven deployment tests.
Assisted-by: gpt-6-astra10 files changed, 188 insertions(+), 22 deletions(-)
nixos/snowglobe.nix+5-4| ... | ... | @@ -5,6 +5,7 @@ let |
| 5 | 5 | xmppNetwork = builtins.fromJSON (builtins.readFile ../service/xmpp/network.json); |
| 6 | 6 | xmppTCPPorts = map (name: xmppNetwork.${name}) [ "c2s" "c2s_tls" "s2s" "turn" "turn_tls" ]; |
| 7 | 7 | xmppUDPPorts = [ xmppNetwork.turn xmppNetwork.turn_tls ]; |
| 8 | irohNetwork = builtins.fromJSON (builtins.readFile ../service/iroh-relay/network.json); | |
| 8 | 9 | encryptionRoots = { |
| 9 | 10 | apps = "apps"; |
| 10 | 11 | clover = "clover"; |
| ... | ... | @@ -24,7 +25,7 @@ in |
| 24 | 25 | networking.hostId = "4fa19ccb"; |
| 25 | 26 | services.nomad.settings.name = config.networking.hostName; |
| 26 | 27 | networking.firewall.allowedTCPPorts = [ 80 443 ] ++ xmppTCPPorts; |
| 27 | networking.firewall.allowedUDPPorts = xmppUDPPorts; | |
| 28 | networking.firewall.allowedUDPPorts = xmppUDPPorts ++ [ irohNetwork.quic ]; | |
| 28 | 29 | networking.firewall.allowedUDPPortRanges = [{ from = xmppNetwork.relay_first; to = xmppNetwork.relay_last; }]; |
| 29 | 30 | networking.firewall.extraCommands = lib.concatMapStrings ({ address, prefixLength, ... }: '' |
| 30 | 31 | iptables -A nixos-fw -i enp4s0 -s ${address}/${toString prefixLength} -p tcp --dport 445 -j nixos-fw-accept |
| ... | ... | @@ -34,7 +35,7 @@ in |
| 34 | 35 | ip6tables -A nixos-fw -i enp4s0 -s fc00::/7 -p tcp --dport 445 -j nixos-fw-accept |
| 35 | 36 | ''; |
| 36 | 37 | |
| 37 | systemd.services.xmpp-nat = { | |
| 38 | systemd.services.public-ports = { | |
| 38 | 39 | wantedBy = [ "multi-user.target" ]; |
| 39 | 40 | wants = [ "network-online.target" ]; |
| 40 | 41 | after = [ "network-online.target" ]; |
| ... | ... | @@ -42,9 +43,9 @@ in |
| 42 | 43 | serviceConfig = { Type = "oneshot"; DynamicUser = true; }; |
| 43 | 44 | script = '' |
| 44 | 45 | description="$(${pkgs.python3}/bin/python3 ${./upnp-discover.py} ${lib.escapeShellArg config.networking.defaultGateway.address})" |
| 45 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe XMPP" -r \ | |
| 46 | exec ${pkgs.miniupnpc}/bin/upnpc -u "$description" -e "Snow Globe services" -r \ | |
| 46 | 47 | ${lib.concatMapStringsSep " " (port: "${toString port} TCP") xmppTCPPorts} \ |
| 47 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} | |
| 48 | ${lib.concatMapStringsSep " " (port: "${toString port} UDP") (xmppUDPPorts ++ [ irohNetwork.quic ] ++ lib.range xmppNetwork.relay_first xmppNetwork.relay_last)} | |
| 48 | 49 | ''; |
| 49 | 50 | }; |
| 50 | 51 |
service/iroh-relay/Dockerfile+5-2| ... | ... | @@ -1,6 +1,9 @@ |
| 1 | FROM docker.io/library/alpine:3.22@sha256:3e9b4b680bfc9fb5269227cffbd6d42be39fbf7c0b908123913864aa4447e764 | |
| 1 | FROM docker.io/library/python:3.13-slim@sha256:37134a49d21d2120e4c4d73bb76f8a4ab9aef31f096f7ec2ead48c2feead4332 | |
| 2 | RUN apt-get update && apt-get install -y --no-install-recommends certbot python3-certbot-dns-cloudflare \ | |
| 3 | && rm -rf /var/lib/apt/lists/* | |
| 2 | 4 | ADD https://github.com/n0-computer/iroh/releases/download/v1.3.0/iroh-relay-v1.3.0-x86_64-unknown-linux-musl.tar.gz /tmp/iroh-relay.tar.gz |
| 3 | 5 | RUN echo '677f4c62342a6ba8044459b5fd4302f2b1dcb8402542072e3a4ade5039bc0b9e /tmp/iroh-relay.tar.gz' | sha256sum -c - \ |
| 4 | 6 | && tar -xzf /tmp/iroh-relay.tar.gz -C /usr/local/bin \ |
| 5 | 7 | && rm /tmp/iroh-relay.tar.gz |
| 6 | ENTRYPOINT ["/usr/local/bin/iroh-relay"] | |
| 8 | COPY run.py network.json /app/ | |
| 9 | ENTRYPOINT ["python3", "/app/run.py"] |
service/iroh-relay/iroh-relay.service+4-1| ... | ... | @@ -5,7 +5,10 @@ After=network-online.target |
| 5 | 5 | |
| 6 | 6 | [Service] |
| 7 | 7 | DynamicUser=true |
| 8 | ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml | |
| 8 | Environment=HTTP_PORT=3340 | |
| 9 | LoadCredential=fullchain.pem:/etc/letsencrypt/live/iroh-relay/fullchain.pem | |
| 10 | LoadCredential=privkey.pem:/etc/letsencrypt/live/iroh-relay/privkey.pem | |
| 11 | ExecStart=/usr/bin/python3 /usr/local/lib/iroh-relay/run.py relay | |
| 9 | 12 | Restart=on-failure |
| 10 | 13 | RestartSec=5 |
| 11 | 14 | NoNewPrivileges=true |
service/iroh-relay/network.json created+1| ... | ... | @@ -0,0 +1 @@ |
| 1 | {"quic":7842} |
service/iroh-relay/r1.caddy+7-1| ... | ... | @@ -1,3 +1,9 @@ |
| 1 | 1 | r1.iroh.paperclover.net { |
| 2 | reverse_proxy 127.0.0.1:3340 | |
| 2 | handle /.well-known/acme-challenge/* { | |
| 3 | root * /var/lib/iroh-acme | |
| 4 | file_server | |
| 5 | } | |
| 6 | handle { | |
| 7 | reverse_proxy 127.0.0.1:3340 | |
| 8 | } | |
| 3 | 9 | } |
service/iroh-relay/relay.toml deleted-2| ... | ... | @@ -1,2 +0,0 @@ |
| 1 | http_bind_addr = "0.0.0.0:3340" | |
| 2 | enable_metrics = false |
service/iroh-relay/run.py created+50| ... | ... | @@ -0,0 +1,50 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | import json | |
| 3 | import os | |
| 4 | from pathlib import Path | |
| 5 | import subprocess | |
| 6 | import sys | |
| 7 | import time | |
| 8 | ||
| 9 | ||
| 10 | os.umask(0o077) | |
| 11 | mode = sys.argv[1] | |
| 12 | tls = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/tls/live/server")) | |
| 13 | ||
| 14 | if mode == "certificates": | |
| 15 | credentials = Path("/tmp/cloudflare.ini") | |
| 16 | credentials.write_text("dns_cloudflare_api_token = " + os.environ.pop("DNS_TOKEN") + "\n") | |
| 17 | command = [ | |
| 18 | "certbot", "certonly", "--non-interactive", "--agree-tos", | |
| 19 | "--email", os.environ["OWNER_EMAIL"], "--dns-cloudflare", | |
| 20 | "--dns-cloudflare-credentials", str(credentials), | |
| 21 | "--dns-cloudflare-propagation-seconds", "30", "--cert-name", "server", | |
| 22 | "--config-dir", "/tls", "--work-dir", "/tmp/certbot-work", "--logs-dir", "/tmp/certbot-logs", | |
| 23 | "--keep-until-expiring", "-d", os.environ["RELAY_HOST"], | |
| 24 | ] | |
| 25 | while True: | |
| 26 | subprocess.run(command, check=not (tls / "fullchain.pem").exists(), timeout=300) | |
| 27 | time.sleep(12 * 60 * 60) | |
| 28 | ||
| 29 | config = Path("/tmp/iroh-relay.toml") | |
| 30 | if mode == "relay": | |
| 31 | for _ in range(180): | |
| 32 | if (tls / "fullchain.pem").exists() and (tls / "privkey.pem").exists(): | |
| 33 | break | |
| 34 | time.sleep(1) | |
| 35 | else: | |
| 36 | raise RuntimeError("Relay certificate was not issued within three minutes") | |
| 37 | quic_port = int(os.environ["QUIC_PORT"]) if "QUIC_PORT" in os.environ else json.loads( | |
| 38 | Path(__file__).with_name("network.json").read_text())["quic"] | |
| 39 | config.write_text( | |
| 40 | 'enable_metrics = false\nenable_quic_addr_discovery = true\n' | |
| 41 | f'http_bind_addr = "127.0.0.1:{int(os.environ["HTTP_PORT"])}"\n' | |
| 42 | '[tls]\ncert_mode = "Reloading"\n' | |
| 43 | f'quic_bind_addr = "0.0.0.0:{quic_port}"\n' | |
| 44 | f'manual_cert_path = {json.dumps(str(tls / "fullchain.pem"))}\n' | |
| 45 | f'manual_key_path = {json.dumps(str(tls / "privkey.pem"))}\n' | |
| 46 | ) | |
| 47 | else: | |
| 48 | raise ValueError("Unknown relay task") | |
| 49 | # --dev leaves HTTPS to Caddy while retaining certificate-backed QUIC discovery. | |
| 50 | os.execv("/usr/local/bin/iroh-relay", ["iroh-relay", "--dev", "--config-path", str(config)]) |
service/iroh-relay/service.pkl+47-11| ... | ... | @@ -1,19 +1,55 @@ |
| 1 | 1 | amends "../../config/Service.pkl" |
| 2 | 2 | |
| 3 | import "../../config/site.pkl" as site | |
| 4 | import "../ddns-updater/service.pkl" as dns | |
| 5 | import "pkl:json" | |
| 6 | ||
| 7 | local network = new json.Parser {}.parse(read("network.json").text) | |
| 8 | local host = read?("prop:hostname") ?? "r2.iroh.\(site.domain)" | |
| 9 | ||
| 3 | 10 | name = "Iroh relay" |
| 4 | 11 | launcher = false |
| 5 | 12 | stageIsolation = "fresh" |
| 13 | requirements { new dns.DnsCredentials {} } | |
| 6 | 14 | |
| 7 | container { | |
| 8 | build = "." | |
| 9 | readOnlyRoot = true | |
| 10 | http { | |
| 11 | containerPort = 3340 | |
| 12 | subdomain = "r2.iroh" | |
| 13 | checkPath = "/healthz" | |
| 15 | containers { | |
| 16 | ["app"] { | |
| 17 | build = "." | |
| 18 | hostNetwork = true | |
| 19 | readOnlyRoot = true | |
| 20 | tmpfs { "/tmp" } | |
| 21 | args { "relay" } | |
| 22 | http { containerPort = 3340; hostname = host; loopback = true; checkPath = "/healthz" } | |
| 23 | tcp { | |
| 24 | new { | |
| 25 | name = "quic" | |
| 26 | containerPort = network.quic | |
| 27 | hostPort = if (site.preview) null else network.quic | |
| 28 | loopback = false | |
| 29 | } | |
| 30 | } | |
| 31 | volumes { ["/tls"] { readOnly = true } } | |
| 32 | envTemplate = """ | |
| 33 | HTTP_PORT={{ env "NOMAD_PORT_app_http" }} | |
| 34 | QUIC_PORT={{ env "NOMAD_PORT_app_quic" }} | |
| 35 | """ | |
| 36 | cpu = 200 | |
| 37 | memory = 256 | |
| 38 | } | |
| 39 | ["certificates"] { | |
| 40 | build = "." | |
| 41 | hostNetwork = true | |
| 42 | readOnlyRoot = true | |
| 43 | tmpfs { "/tmp" } | |
| 44 | lifecycle = "prestartSidecar" | |
| 45 | args { "certificates" } | |
| 46 | volumes { ["/tls"] {} } | |
| 47 | env { | |
| 48 | ["RELAY_HOST"] = host | |
| 49 | ["OWNER_EMAIL"] = site.ownerEmail | |
| 50 | ["DNS_TOKEN"] = "${secret.dns.api_token}" | |
| 51 | } | |
| 52 | cpu = 100 | |
| 53 | memory = 256 | |
| 14 | 54 | } |
| 15 | args { "--config-path"; "/etc/iroh-relay.toml" } | |
| 16 | volumes { ["/etc/iroh-relay.toml"] { config = "relay.toml" } } | |
| 17 | cpu = 200 | |
| 18 | memory = 256 | |
| 19 | 55 | } |
tools/iroh-relay.md created+68| ... | ... | @@ -0,0 +1,68 @@ |
| 1 | # Iroh relays | |
| 2 | ||
| 3 | Clients use `https://r1.iroh.paperclover.net` and | |
| 4 | `https://r2.iroh.paperclover.net`. Caddy terminates HTTPS on TCP 443; Iroh serves | |
| 5 | QUIC address discovery on UDP 7842. TCP 80 remains available for ACME. | |
| 6 | These are public relays without an application login. | |
| 7 | ||
| 8 | The upstream binary version and checksum live in | |
| 9 | [`Dockerfile`](../service/iroh-relay/Dockerfile). Both machines use | |
| 10 | [`run.py`](../service/iroh-relay/run.py) and | |
| 11 | [`network.json`](../service/iroh-relay/network.json). | |
| 12 | Iroh's `--dev` flag disables its HTTPS listener behind Caddy, while its | |
| 13 | `Reloading` certificate resolver still supplies valid TLS for UDP discovery. | |
| 14 | ||
| 15 | ## Snow Globe | |
| 16 | ||
| 17 | [`service.pkl`](../service/iroh-relay/service.pkl) declares the relay and its | |
| 18 | certificate sidecar. The sidecar obtains the Cloudflare credential through the | |
| 19 | DDNS provider, keeps it in a private tmpfs file, and checks certificate renewal | |
| 20 | every twelve hours. Only the certificate volume is shared with the relay. | |
| 21 | The relay reloads renewed certificates without restarting connections. | |
| 22 | ||
| 23 | Stage and deploy with the normal `tools/deploy.py` workflow. Fresh previews have | |
| 24 | their own certificate storage and dynamically allocated UDP port; they do not | |
| 25 | bind production's port. The host's `public-ports.service` maintains the production | |
| 26 | router mapping; the firewall declaration is in `nixos/snowglobe.nix`. | |
| 27 | ||
| 28 | ## paperclover.net | |
| 29 | ||
| 30 | Connect with `ssh root@paperclover.net`; the hostname is `usw-server-1`. | |
| 31 | The checksum-verified upstream binary is installed at `/usr/local/bin/iroh-relay`. | |
| 32 | Install `run.py` and `network.json` into `/usr/local/lib/iroh-relay/`, and | |
| 33 | [`iroh-relay.service`](../service/iroh-relay/iroh-relay.service) into | |
| 34 | `/etc/systemd/system/`. | |
| 35 | ||
| 36 | [`r1.caddy`](../service/iroh-relay/r1.caddy) is installed as | |
| 37 | `/etc/caddy/Caddyfile.d/iroh-relay.caddy`, explicitly imported by the main | |
| 38 | Caddyfile. Validate before reloading Caddy. The ACME webroot is | |
| 39 | `/var/lib/iroh-acme`; Caddy must be able to read it. | |
| 40 | ||
| 41 | After Caddy's HTTPS certificate is ready, issue the discovery certificate: | |
| 42 | ||
| 43 | ```sh | |
| 44 | certbot certonly --non-interactive --agree-tos --email account@paperclover.net \ | |
| 45 | --webroot -w /var/lib/iroh-acme --cert-name iroh-relay \ | |
| 46 | -d r1.iroh.paperclover.net \ | |
| 47 | --deploy-hook 'systemctl try-restart iroh-relay.service' | |
| 48 | systemctl daemon-reload | |
| 49 | systemctl enable --now iroh-relay.service certbot-renew.timer | |
| 50 | firewall-cmd --add-port=7842/udp | |
| 51 | firewall-cmd --permanent --add-port=7842/udp | |
| 52 | ``` | |
| 53 | ||
| 54 | Systemd loads the certificate and key as private service credentials. Certbot's | |
| 55 | renewal hook restarts the service to refresh those credentials. The key stays | |
| 56 | root-only under `/etc/letsencrypt`. | |
| 57 | ||
| 58 | ## Verification | |
| 59 | ||
| 60 | `GET /healthz` reports the relay version, but does not test packet forwarding. | |
| 61 | Use two Iroh 1.3 endpoints with `clear_ip_transports()` and a custom relay map; | |
| 62 | transfer and verify a payload in both directions. For UDP, connect a | |
| 63 | `iroh_relay::quic::QuicClient` with normal CA validation and await its | |
| 64 | `observed_external_addr()` stream. Test from outside the server's network. | |
| 65 | ||
| 66 | Renewal can be exercised on r1 with | |
| 67 | `certbot renew --cert-name iroh-relay --dry-run --run-deploy-hooks`. | |
| 68 | Inspect the relay's restart count and health afterward. |
tools/xmpp.md+1-1| ... | ... | @@ -24,7 +24,7 @@ Production firewall rules are declared in `nixos/snowglobe.nix`. The DNS provide |
| 24 | 24 | provisions client, direct-TLS client, and federation SRV records pointing at |
| 25 | 25 | `xmpp.paperclover.net`; the website keeps its existing apex address. Router |
| 26 | 26 | forwarding must send the TCP listeners, UDP TURN listeners, and UDP relay range |
| 27 | to Snow Globe. `xmpp-nat.service` refreshes UPnP mappings on startup and every five | |
| 27 | to Snow Globe. `public-ports.service` refreshes UPnP mappings on startup and every five | |
| 28 | 28 | minutes, discovering the current description URL with a bounded unicast SSDP |
| 29 | 29 | request to the configured gateway on port 1900. Replies and HTTP description |
| 30 | 30 | hosts must match that gateway; discovery failure fails the refresh. Multicast |