Behind a proxy: plain HTTP for the relay, the certificate for QUIC alone

deploy/relay.toml

@@ -9,6 +9,11 @@ hostname = "relay-hk1.baocode.dev"
 quic = true
 metrics = true
 
+# Behind a reverse proxy that has ports 80/443 (Caddy, say) and ends TLS:
+# the relay answers in plain HTTP on [bind] http, and [tls] (mode "files",
+# the proxy's certificate) is for QUIC alone. See docs/operations.md.
+# behind_proxy = true
+
 [bind]
 http = "[::]:80"          # captive portal probes (ACME itself uses TLS-ALPN on 443)
 https = "[::]:443"        # the relay (WebSocket over TLS)

docs/operations.md

@@ -2,7 +2,7 @@
 
 ## 机器与域名
 
-- 至少两台，放在不同地区。默认内置的是 `relay-hk1.baocode.dev`（香港，线路优先选 CN2 GIA / CMI）和 `relay-jp1.baocode.dev`（日本）。改名或加机器时，要同步改客户端的内置列表（monad `packages/bao_link/lib/src/relays.dart`）。
+- 至少两台，放在不同地区。默认内置的是 `relay-hk1.baocode.dev`（香港，线路优先选 CN2 GIA / CMI）和 `relay-jp1.baocode.dev`（日本），还没部署；已上线的是 `relay-us1.baocode.dev`（洛杉矶，搬瓦工，在 Caddy 后面，见下文）。改名或加机器时，要同步改客户端的内置列表（monad `packages/bao_link/lib/src/relays.dart`）。
 - 配置：1 核 / 1 GB 就能跑，瓶颈是带宽。按月流量计费的套餐，看下面的「带宽估算」。
 - DNS：每台一条 A 记录（有 IPv6 就再加 AAAA），**直接指向机器，不要开 Cloudflare 代理（橙云）**，因为 UDP 7842 和 WebSocket 都要直连。
 - 端口：TCP 80、TCP 443、UDP 7842。云厂商的安全组也要放行。
@@ -44,6 +44,23 @@
 
    推荐用 `--network host`，因为 UDP 打洞要用到真实的源地址和端口。
 
+## 机器上已有 Caddy 占着 80/443
+
+（`relay-us1` 就是这样：搬瓦工上的 `/root/docker/docker-compose.yml`。）域名和证书交给 Caddy，中继在后面：
+
+- `relay.toml` 里 `behind_proxy = true`：中继在 `[bind] http` 上用明文提供服务，不绑 `https`；`[tls]` 用 `mode = "files"` 指向 Caddy 的证书，只给 UDP 7842 上的 QUIC 地址发现用（每小时重读一次，Caddy 续期后自动跟上）。
+- Caddyfile：
+
+  ```
+  relay-us1.baocode.dev {
+      reverse_proxy baocode-relay:3340
+  }
+  ```
+
+- compose 里中继和 Caddy 在同一个网络，只映射 `7842:7842/udp`；把 Caddy 的 `./caddy/data/caddy/certificates` 只读挂进去。Docker 映射的 UDP 端口走 iptables DNAT，源地址不变，QUIC 地址发现看到的是设备的真实地址。
+- QUIC 用的内核缓冲区（`net.core.rmem_max` / `wmem_max`）是整机的，在宿主机的 `/etc/sysctl.d/` 里设，容器里设不了。
+- 升级：把新的 `baocode-relay` 二进制放进构建目录，`docker compose up -d --build baocode-relay`。
+
 ## 升级
 
 下载新版本的包，再执行一次 `install.sh`。配置、封禁名单和证书都会保留。重启时已连着的设备会断开几秒，然后自动重连。如果有另一台中继，设备在这期间会走那一台。

src/config.rs

@@ -30,6 +30,11 @@ pub struct Config {
     /// QUIC address discovery (UDP), which hole punching needs. Needs TLS.
     pub quic: bool,
     pub metrics: bool,
+    /// A reverse proxy (Caddy, say) ends TLS for the relay and passes it on
+    /// to `bind.http`: the relay answers there in plain HTTP and `https` is
+    /// not bound. `tls` (files, the proxy's certificate) is then for QUIC
+    /// alone.
+    pub behind_proxy: bool,
 }
 
 impl Default for Config {
@@ -43,6 +48,7 @@ impl Default for Config {
             access: AccessFiles::default(),
             quic: true,
             metrics: true,
+            behind_proxy: false,
         }
     }
 }
@@ -189,6 +195,9 @@ impl Config {
         {
             bail!("letsencrypt needs a hostname");
         }
+        if self.behind_proxy && matches!(self.tls, Some(Tls::Letsencrypt { .. })) {
+            bail!("behind_proxy: the proxy gets the certificate; give it to the relay as files");
+        }
         if self.limits.bytes_per_second == 0 && self.limits.burst_bytes != 0 {
             bail!("burst_bytes needs bytes_per_second");
         }
@@ -248,5 +257,11 @@ mod tests {
         assert!(Config::parse("quic = true").is_err());
         assert!(Config::parse("[tls]\nmode = \"letsencrypt\"\ncontact = \"a\"").is_err());
         assert!(Config::parse("typo = 1").is_err());
+        assert!(
+            Config::parse(
+                "hostname = \"a\"\nbehind_proxy = true\n[tls]\nmode = \"letsencrypt\"\ncontact = \"a\""
+            )
+            .is_err()
+        );
     }
 }

src/lib.rs

@@ -12,7 +12,7 @@ use std::{
     time::Duration,
 };
 
-use anyhow::{Context, Result, anyhow};
+use anyhow::{Context, Result, anyhow, bail};
 use iroh_metrics::Registry;
 use iroh_relay::server::{
     self as relay, AcmeConfig, CertConfig, QuicConfig, RelayConfig, RelayService, Server,
@@ -49,12 +49,23 @@ impl Relay {
         relay_config.access = Arc::new(gate.clone());
         let mut quic = None;
         if let Some(tls) = &config.tls {
-            relay_config.tls = Some(TlsConfig::new(
-                config.bind.https,
-                certs(&config, tls).await?,
-            ));
-            if config.quic {
-                quic = Some(QuicConfig::new(config.bind.quic));
+            let certs = certs(&config, tls).await?;
+            if config.behind_proxy {
+                // The proxy ends TLS for the relay, which answers it on
+                // bind.http: the certificate is QUIC's alone.
+                if config.quic {
+                    let CertConfig::Manual { server_config } = certs else {
+                        bail!("behind_proxy needs tls mode files");
+                    };
+                    let mut quic_config = QuicConfig::new(config.bind.quic);
+                    quic_config.server_config = Some(server_config);
+                    quic = Some(quic_config);
+                }
+            } else {
+                relay_config.tls = Some(TlsConfig::new(config.bind.https, certs));
+                if config.quic {
+                    quic = Some(QuicConfig::new(config.bind.quic));
+                }
             }
         }
         let mut server_config = ServerConfig::default();

tests/relay.rs

@@ -385,3 +385,81 @@ async fn over_tls_with_metrics_as_deployed() {
     // No device ids in there.
     assert!(!text.contains(&key(1).public().to_string()));
 }
+
+#[tokio::test]
+async fn behind_a_proxy_plain_http_with_quic_on_the_certificate() {
+    let dir = tempfile::tempdir().unwrap();
+    let cert = rcgen::generate_simple_self_signed(vec!["127.0.0.1".into()]).unwrap();
+    let cert_path = dir.path().join("cert.pem");
+    let key_path = dir.path().join("key.pem");
+    std::fs::write(&cert_path, cert.cert.pem()).unwrap();
+    std::fs::write(&key_path, cert.signing_key.serialize_pem()).unwrap();
+    let config = Config::parse(&format!(
+        r#"
+        hostname = "127.0.0.1"
+        behind_proxy = true
+        [bind]
+        http = "127.0.0.1:0"
+        quic = "127.0.0.1:0"
+        metrics = "127.0.0.1:0"
+        [tls]
+        mode = "files"
+        cert = "{}"
+        key = "{}"
+        "#,
+        cert_path.display(),
+        key_path.display()
+    ))
+    .unwrap();
+    let relay = Relay::spawn(config, None).await.unwrap();
+    // What the proxy passes on to: the relay in plain HTTP, no HTTPS of its own.
+    assert!(relay.https_addr().is_none());
+    let quic_port = relay.quic_addr().unwrap().port();
+    let url: RelayUrl = format!("http://{}", relay.http_addr().unwrap())
+        .parse()
+        .unwrap();
+    let relays = [url.clone()];
+
+    // An end with IP transport, so it asks the relay's QUIC for its address.
+    let roots = iroh_relay::tls::CaTlsConfig::custom_roots([cert.cert.der().clone()]);
+    let map = iroh::RelayMap::empty();
+    map.insert(
+        url.clone(),
+        Arc::new(RelayConfig::new(
+            url,
+            Some(iroh_relay::RelayQuicConfig::new(quic_port)),
+        )),
+    );
+    let probing = Endpoint::builder(presets::Minimal)
+        .secret_key(key(3))
+        .relay_mode(RelayMode::Custom(map))
+        .ca_tls_config(roots)
+        .bind()
+        .await
+        .unwrap();
+    let metrics = format!("http://{}/metrics", relay.metrics_addr().unwrap());
+    timeout(Duration::from_secs(20), async {
+        loop {
+            let text = reqwest::get(&metrics).await.unwrap().text().await.unwrap();
+            if text
+                .lines()
+                .any(|l| l.starts_with("relayserver_qad_connections_total ") && !l.ends_with(" 0"))
+            {
+                break;
+            }
+            tokio::time::sleep(Duration::from_millis(100)).await;
+        }
+    })
+    .await
+    .expect("a QUIC address discovery round trip");
+    drop(probing);
+
+    let server = endpoint(key(1), &relays).await;
+    let client = endpoint(key(2), &relays).await;
+    echo(&server);
+    timeout(Duration::from_secs(10), server.online())
+        .await
+        .unwrap();
+    let connection = client.connect(addr(&server, &relays), ALPN).await.unwrap();
+    round_trip(&connection, 100_000).await.unwrap();
+}