From 2c31b73012a0e302d24d3d0b350574386b7aca7c Mon Sep 17 00:00:00 2001 From: leo Date: Sat, 31 Oct 2026 12:05:11 +0800 Subject: [PATCH] Behind a proxy: plain HTTP for the relay, the certificate for QUIC alone For a machine where Caddy already has ports 80/443 and the domain: `behind_proxy = true` serves the relay on [bind] http and gives [tls] (the proxy's certificate, as files) to QUIC address discovery only. Co-Authored-By: Claude Opus 5.5 --- deploy/relay.toml | 5 +++ docs/operations.md | 19 ++++++++++- src/config.rs | 15 +++++++++ src/lib.rs | 25 ++++++++++----- tests/relay.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 8 deletions(-) diff --git a/deploy/relay.toml b/deploy/relay.toml index ee67a7d..2c0546b 100644 --- a/deploy/relay.toml +++ b/deploy/relay.toml @@ -9,6 +9,11 @@ hostname = "relay-hk1.baocode.dev" quic = true metrics = true +# Behind a reverse proxy that has ports 80/443 (Caddy, say) and ends TLS: +# the relay answers in plain HTTP on [bind] http, and [tls] (mode "files", +# the proxy's certificate) is for QUIC alone. See docs/operations.md. +# behind_proxy = true + [bind] http = "[::]:80" # captive portal probes (ACME itself uses TLS-ALPN on 443) https = "[::]:443" # the relay (WebSocket over TLS) diff --git a/docs/operations.md b/docs/operations.md index 8ef26b9..3f2eeed 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -2,7 +2,7 @@ ## 机器与域名 -- 至少两台，放在不同地区。默认内置的是 `relay-hk1.baocode.dev`（香港，线路优先选 CN2 GIA / CMI）和 `relay-jp1.baocode.dev`（日本）。改名或加机器时，要同步改客户端的内置列表（monad `packages/bao_link/lib/src/relays.dart`）。 +- 至少两台，放在不同地区。默认内置的是 `relay-hk1.baocode.dev`（香港，线路优先选 CN2 GIA / CMI）和 `relay-jp1.baocode.dev`（日本），还没部署；已上线的是 `relay-us1.baocode.dev`（洛杉矶，搬瓦工，在 Caddy 后面，见下文）。改名或加机器时，要同步改客户端的内置列表（monad `packages/bao_link/lib/src/relays.dart`）。 - 配置：1 核 / 1 GB 就能跑，瓶颈是带宽。按月流量计费的套餐，看下面的「带宽估算」。 - DNS：每台一条 A 记录（有 IPv6 就再加 AAAA），**直接指向机器，不要开 Cloudflare 代理（橙云）**，因为 UDP 7842 和 WebSocket 都要直连。 - 端口：TCP 80、TCP 443、UDP 7842。云厂商的安全组也要放行。 @@ -44,6 +44,23 @@ 推荐用 `--network host`，因为 UDP 打洞要用到真实的源地址和端口。 +## 机器上已有 Caddy 占着 80/443 + +（`relay-us1` 就是这样：搬瓦工上的 `/root/docker/docker-compose.yml`。）域名和证书交给 Caddy，中继在后面： + +- `relay.toml` 里 `behind_proxy = true`：中继在 `[bind] http` 上用明文提供服务，不绑 `https`；`[tls]` 用 `mode = "files"` 指向 Caddy 的证书，只给 UDP 7842 上的 QUIC 地址发现用（每小时重读一次，Caddy 续期后自动跟上）。 +- Caddyfile： + + ``` + relay-us1.baocode.dev { + reverse_proxy baocode-relay:3340 + } + ``` + +- compose 里中继和 Caddy 在同一个网络，只映射 `7842:7842/udp`；把 Caddy 的 `./caddy/data/caddy/certificates` 只读挂进去。Docker 映射的 UDP 端口走 iptables DNAT，源地址不变，QUIC 地址发现看到的是设备的真实地址。 +- QUIC 用的内核缓冲区（`net.core.rmem_max` / `wmem_max`）是整机的，在宿主机的 `/etc/sysctl.d/` 里设，容器里设不了。 +- 升级：把新的 `baocode-relay` 二进制放进构建目录，`docker compose up -d --build baocode-relay`。 + ## 升级 下载新版本的包，再执行一次 `install.sh`。配置、封禁名单和证书都会保留。重启时已连着的设备会断开几秒，然后自动重连。如果有另一台中继，设备在这期间会走那一台。 diff --git a/src/config.rs b/src/config.rs index 9a4ca16..85c3de8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -30,6 +30,11 @@ pub struct Config { /// QUIC address discovery (UDP), which hole punching needs. Needs TLS. pub quic: bool, pub metrics: bool, + /// A reverse proxy (Caddy, say) ends TLS for the relay and passes it on + /// to `bind.http`: the relay answers there in plain HTTP and `https` is + /// not bound. `tls` (files, the proxy's certificate) is then for QUIC + /// alone. + pub behind_proxy: bool, } impl Default for Config { @@ -43,6 +48,7 @@ impl Default for Config { access: AccessFiles::default(), quic: true, metrics: true, + behind_proxy: false, } } } @@ -189,6 +195,9 @@ impl Config { { bail!("letsencrypt needs a hostname"); } + if self.behind_proxy && matches!(self.tls, Some(Tls::Letsencrypt { .. })) { + bail!("behind_proxy: the proxy gets the certificate; give it to the relay as files"); + } if self.limits.bytes_per_second == 0 && self.limits.burst_bytes != 0 { bail!("burst_bytes needs bytes_per_second"); } @@ -248,5 +257,11 @@ mod tests { assert!(Config::parse("quic = true").is_err()); assert!(Config::parse("[tls]\nmode = \"letsencrypt\"\ncontact = \"a\"").is_err()); assert!(Config::parse("typo = 1").is_err()); + assert!( + Config::parse( + "hostname = \"a\"\nbehind_proxy = true\n[tls]\nmode = \"letsencrypt\"\ncontact = \"a\"" + ) + .is_err() + ); } } diff --git a/src/lib.rs b/src/lib.rs index bcfca2f..4be06f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ use std::{ time::Duration, }; -use anyhow::{Context, Result, anyhow}; +use anyhow::{Context, Result, anyhow, bail}; use iroh_metrics::Registry; use iroh_relay::server::{ self as relay, AcmeConfig, CertConfig, QuicConfig, RelayConfig, RelayService, Server, @@ -49,12 +49,23 @@ impl Relay { relay_config.access = Arc::new(gate.clone()); let mut quic = None; if let Some(tls) = &config.tls { - relay_config.tls = Some(TlsConfig::new( - config.bind.https, - certs(&config, tls).await?, - )); - if config.quic { - quic = Some(QuicConfig::new(config.bind.quic)); + let certs = certs(&config, tls).await?; + if config.behind_proxy { + // The proxy ends TLS for the relay, which answers it on + // bind.http: the certificate is QUIC's alone. + if config.quic { + let CertConfig::Manual { server_config } = certs else { + bail!("behind_proxy needs tls mode files"); + }; + let mut quic_config = QuicConfig::new(config.bind.quic); + quic_config.server_config = Some(server_config); + quic = Some(quic_config); + } + } else { + relay_config.tls = Some(TlsConfig::new(config.bind.https, certs)); + if config.quic { + quic = Some(QuicConfig::new(config.bind.quic)); + } } } let mut server_config = ServerConfig::default(); diff --git a/tests/relay.rs b/tests/relay.rs index f27a66b..79944aa 100644 --- a/tests/relay.rs +++ b/tests/relay.rs @@ -385,3 +385,81 @@ async fn over_tls_with_metrics_as_deployed() { // No device ids in there. assert!(!text.contains(&key(1).public().to_string())); } + +#[tokio::test] +async fn behind_a_proxy_plain_http_with_quic_on_the_certificate() { + let dir = tempfile::tempdir().unwrap(); + let cert = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap(); + let cert_path = dir.path().join("cert.pem"); + let key_path = dir.path().join("key.pem"); + std::fs::write(&cert_path, cert.cert.pem()).unwrap(); + std::fs::write(&key_path, cert.signing_key.serialize_pem()).unwrap(); + let config = Config::parse(&format!( + r#" + hostname = "127.0.0.1" + behind_proxy = true + [bind] + http = "127.0.0.1:0" + quic = "127.0.0.1:0" + metrics = "127.0.0.1:0" + [tls] + mode = "files" + cert = "{}" + key = "{}" + "#, + cert_path.display(), + key_path.display() + )) + .unwrap(); + let relay = Relay::spawn(config, None).await.unwrap(); + // What the proxy passes on to: the relay in plain HTTP, no HTTPS of its own. + assert!(relay.https_addr().is_none()); + let quic_port = relay.quic_addr().unwrap().port(); + let url: RelayUrl = format!("http://{}", relay.http_addr().unwrap()) + .parse() + .unwrap(); + let relays = [url.clone()]; + + // An end with IP transport, so it asks the relay's QUIC for its address. + let roots = iroh_relay::tls::CaTlsConfig::custom_roots([cert.cert.der().clone()]); + let map = iroh::RelayMap::empty(); + map.insert( + url.clone(), + Arc::new(RelayConfig::new( + url, + Some(iroh_relay::RelayQuicConfig::new(quic_port)), + )), + ); + let probing = Endpoint::builder(presets::Minimal) + .secret_key(key(3)) + .relay_mode(RelayMode::Custom(map)) + .ca_tls_config(roots) + .bind() + .await + .unwrap(); + let metrics = format!("http://{}/metrics", relay.metrics_addr().unwrap()); + timeout(Duration::from_secs(20), async { + loop { + let text = reqwest::get(&metrics).await.unwrap().text().await.unwrap(); + if text + .lines() + .any(|l| l.starts_with("relayserver_qad_connections_total ") && !l.ends_with(" 0")) + { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("a QUIC address discovery round trip"); + drop(probing); + + let server = endpoint(key(1), &relays).await; + let client = endpoint(key(2), &relays).await; + echo(&server); + timeout(Duration::from_secs(10), server.online()) + .await + .unwrap(); + let connection = client.connect(addr(&server, &relays), ALPN).await.unwrap(); + round_trip(&connection, 100_000).await.unwrap(); +}