Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,11 @@ full cold-read/resource acceptance gates listed in `docs/spec/README.md`.
crate shared with the webhook bridge, whose bucket cursor (`events/cursor.json`) is unchanged. Wake-ups
arrive over NATS and the periodic sweep stays the backstop; the service API is the NATS micro protocol, no HTTP.

- **D51 (2026-10-27): The JetStream sink ships as its own `walgit-nats` binary.** This amends D9's one-binary
rule for this sink only: it carries no HTTP server or web UI and depends on neither `walgit-server` nor
`walgit-cli`. It shares config loading and telemetry with the other binaries through `walgit-telemetry`, and
reads the same `walgit.toml` as serve or a smaller file holding only the sections it uses (`docs/EVENTS.md`).

## 5. Working rules

- **No backwards compatibility (pre-1.0, banner at top):** change the shape and delete the old one in the same
Expand Down
4 changes: 3 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ ARG WALGIT_BUILD_SHA=dev
ENV WALGIT_BUILD_SHA=${WALGIT_BUILD_SHA}
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/src/target \
cargo build --release --locked -p walgit-cli \
cargo build --release --locked -p walgit-cli -p walgit-nats \
&& install -D target/release/walgit /out/bin/walgit \
&& install -D target/release/walgit-server /out/bin/walgit-server
&& install -D target/release/walgit-server /out/bin/walgit-server \
&& install -D target/release/walgit-nats /out/bin/walgit-nats

# ---- 3. runtime -----------------------------------------------------------------------------
# trixie ships git 2.47+: walgit wants >= 2.47 on the server (pack.writeReverseIndex,
Expand All @@ -50,7 +51,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends git git-lfs cor
&& git --version
RUN useradd --uid 1000 --create-home --shell /bin/sh walgit \
&& mkdir -p /etc/walgit /var/lib/walgit && chown walgit:walgit /var/lib/walgit
COPY --from=build /out/bin/walgit /out/bin/walgit-server /usr/local/bin/
COPY --from=build /out/bin/walgit /out/bin/walgit-server /out/bin/walgit-nats /usr/local/bin/
COPY walgit.example.toml /etc/walgit/walgit.toml
ENV RUST_LOG=info,walgit=debug \
WALGIT_CONFIG=/etc/walgit/walgit.toml \
Expand Down
1 change: 0 additions & 1 deletion crates/walgit-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ walgit-telemetry.workspace = true
walgit-git.workspace = true
walgit-wal.workspace = true
walgit-server.workspace = true
walgit-nats.workspace = true
gix-hash.workspace = true
anyhow.workspace = true
tokio.workspace = true
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-cli/src/bin/walgit.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! `walgit`: the full CLI (serve | compact | repo | wal | nats | synth | import | mirror | config).
//! `walgit`: the full CLI (serve | compact | repo | wal | synth | import | mirror | config).
fn main() -> anyhow::Result<()> {
walgit_cli::main()
}
3 changes: 0 additions & 3 deletions crates/walgit-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,6 @@ enum Command {
#[command(subcommand)]
action: RepoAction,
},
/// Publish the WAL's ref events into NATS `JetStream` (`[events.nats]`, `docs/EVENTS.md`).
Nats,
/// WAL inspection and rewind.
Wal {
#[command(subcommand)]
Expand Down Expand Up @@ -441,7 +439,6 @@ async fn dispatch(command: Command, cfg: Config) -> Result<()> {
} => compact::run(repo, all, once, base, &cfg).await,
Command::Repo { action } => repo::run(action, &cfg).await,
Command::Wal { action } => wal_cmd::run(action, &cfg).await,
Command::Nats => walgit_nats::run(cfg.clone()).await,
Command::Mirror {
from,
to,
Expand Down
4 changes: 2 additions & 2 deletions crates/walgit-config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ pub struct EventsConfig {
/// warns when a sweep finds unpublished entries. `0` = off.
#[serde(with = "humantime_serde")]
pub sweep_interval: Duration,
/// The `JetStream` publisher (`walgit nats`, `docs/EVENTS.md`). Read only by that subcommand.
/// The `JetStream` publisher (`walgit-nats`, `docs/EVENTS.md`). Read only by that binary.
pub nats: NatsEventsConfig,
}

Expand All @@ -663,7 +663,7 @@ pub enum NatsStart {
Head,
}

/// `[events.nats]`: `walgit nats` publishes the bridge's `ref` events into `JetStream`.
/// `[events.nats]`: `walgit-nats` publishes the bridge's `ref` events into `JetStream`.
/// The resume cursor is the last message on each repository's subject, so there is no cursor
/// object in the bucket.
#[derive(Debug, Clone, Serialize, Deserialize)]
Expand Down
7 changes: 7 additions & 0 deletions crates/walgit-nats/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[[bin]]
name = "walgit-nats"
path = "src/bin/walgit-nats.rs"

[dependencies]
walgit-events.workspace = true
walgit-config.workspace = true
walgit-store.workspace = true
walgit-git.workspace = true
walgit-wal.workspace = true
walgit-telemetry.workspace = true
clap.workspace = true
mimalloc.workspace = true
async-nats.workspace = true
nkeys.workspace = true
percent-encoding.workspace = true
Expand Down
32 changes: 32 additions & 0 deletions crates/walgit-nats/src/bin/walgit-nats.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
//! `walgit-nats --config walgit.toml`: the WAL's `ref` events published into NATS `JetStream`
//! (D50, D51), without the server or the web UI. `--config` may name the file `walgit serve`
//! reads or a smaller one holding only `[store]`, `[cache]`, `[wal]`, `[events]` and `[telemetry]`.

use std::path::PathBuf;
use std::sync::Arc;

use clap::Parser;

#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;

#[derive(Parser)]
#[command(
name = "walgit-nats",
version = walgit_telemetry::BUILD_SHA,
about = "Publish walgit's ref events into NATS JetStream"
)]
struct Cli {
/// Path to the configuration file.
#[arg(long, env = "WALGIT_CONFIG", default_value = "walgit.toml")]
config: PathBuf,
}

fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
let cfg = walgit_telemetry::startup(&cli.config, "walgit-nats");
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
.block_on(walgit_nats::run(Arc::new(cfg)))
}
4 changes: 2 additions & 2 deletions crates/walgit-nats/src/env.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! How `walgit nats` reaches NATS, read from the same `NATS_*` environment variables the `nats` CLI
//! How `walgit-nats` reaches NATS, read from the same `NATS_*` environment variables the `nats` CLI
//! reads and combined the way it combines them, so one shell environment drives both and secrets
//! never sit in `walgit.toml`.

Expand Down Expand Up @@ -107,7 +107,7 @@ impl NatsEnv {
for unsupported in [ENV_NATS_CONTEXT, ENV_NATS_SOCKS_PROXY] {
anyhow::ensure!(
var(unsupported).is_none(),
"{unsupported} is set but walgit nats does not support it; unset it and use the other NATS_* variables"
"{unsupported} is set but walgit-nats does not support it; unset it and use the other NATS_* variables"
);
}
let home = var("HOME");
Expand Down
6 changes: 3 additions & 3 deletions crates/walgit-nats/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! `walgit nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with
//! `walgit-nats`: the WAL's `ref` events (`docs/EVENTS.md`) published into NATS `JetStream`, with
//! the cursor in the stream, wake-ups over NATS and a `$SRV` service API instead of HTTP.
//! The event model is shared with the webhook bridge through `walgit-events`.

Expand Down Expand Up @@ -50,7 +50,7 @@ pub async fn open_stream(
) -> anyhow::Result<stream::Stream> {
let stream = js.get_stream(&cfg.stream).await.with_context(|| {
format!(
"stream {} (walgit nats never creates it; provision it as docs/EVENTS.md shows)",
"stream {} (walgit-nats never creates it; provision it as docs/EVENTS.md shows)",
cfg.stream
)
})?;
Expand Down Expand Up @@ -98,7 +98,7 @@ pub fn publisher_config(cfg: &Config) -> Config {
cfg
}

/// `walgit nats`: publish until SIGTERM/ctrl-c, or fail when a worker stops.
/// `walgit-nats`: publish until SIGTERM/ctrl-c, or fail when a worker stops.
pub async fn run(cfg: Arc<Config>) -> anyhow::Result<()> {
let nats = &cfg.events.nats;
let prefix = SubjectPrefix::new(&nats.subject_prefix)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-nats/src/stream_shape.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! What `walgit nats` needs from the stream it publishes into. The operator provisions the stream
//! What `walgit-nats` needs from the stream it publishes into. The operator provisions the stream
//! (`docs/EVENTS.md`); walgit only checks it, so it never needs the permission to create one.

use std::fmt;
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-nats/tests/env.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! `walgit nats` connects with whatever the `NATS_*` variables resolve to, against a real
//! `walgit-nats` connects with whatever the `NATS_*` variables resolve to, against a real
//! `nats-server` that requires it.

use std::time::Duration;
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-nats/tests/jetstream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ impl Rig {
)
}

/// What `walgit nats` runs: its own registry over the same store (it never sees the
/// What `walgit-nats` runs: its own registry over the same store (it never sees the
/// writer's pushes except through the bucket) on the operator's `[wal]` settings, as
/// `run` adapts them. Returns the registry's cache dir guard too.
async fn operator_publisher(
Expand Down
2 changes: 1 addition & 1 deletion crates/walgit-server/tests/events_nats.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Test fixtures use panics to fail the test, including shared helper functions.
#![allow(clippy::indexing_slicing, clippy::unwrap_used)]

//! Events (docs/EVENTS.md, `walgit nats`): real `git push`es against the harness server, published
//! Events (docs/EVENTS.md, `walgit-nats`): real `git push`es against the harness server, published
//! into a real `nats-server -js` by a `walgit_nats` publisher over the same registry. Skips (passes)
//! when `nats-server` is not on PATH.
mod harness;
Expand Down
27 changes: 15 additions & 12 deletions docs/EVENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,18 @@ webhook_secret = "…" # env: WALGIT__EVENTS__WEBHOOK_SECRET
sweep_interval = "5m"
```

## Delivery: NATS JetStream (`walgit nats`)
## Delivery: NATS JetStream (`walgit-nats`)

`walgit nats` is a second sink for the same `ref` events, a subcommand of the normal CLI
(`walgit nats --config walgit.toml`, `[events.nats]`) built on the `walgit-nats` crate, which depends on
`walgit-events` (the shared event model) and never on `walgit-server`. The webhook bridge is unchanged and the two can run side by side.
`walgit-nats` is a second sink for the same `ref` events, its own binary (`walgit-nats --config walgit.toml`,
`[events.nats]`) built from the `walgit-nats` crate. It depends on `walgit-events` (the shared event model) and
`walgit-telemetry` (config loading and logging), never on `walgit-server` or `walgit-cli`, so it carries no HTTP
server and no web UI. The webhook bridge is unchanged and the two can run side by side.
Like the bridge it is a reader of the WAL, never a step of a push (D32). It keeps its local copies under
`<cache.dir>/.nats`, so it can share a `walgit.toml` and a host with `walgit serve` without touching serve's.
`--config` (or `WALGIT_CONFIG`) may name that same file, or a second, smaller one holding only `[store]`,
`[cache]`, `[wal]`, `[events]` and `[telemetry]`.

- **The operator provisions the stream; `walgit nats` never creates it**, so its credentials need no
- **The operator provisions the stream; `walgit-nats` never creates it**, so its credentials need no
stream-admin permission, only stream info and message reads on the stream and publishes on its subjects.
The stream it expects, for the default `stream` and `subject_prefix`:

Expand Down Expand Up @@ -152,7 +155,7 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). It keep
shapes as `POST /_events/notify` (S3 event JSON, a GCS envelope, `{"key": ...}`, `{"repo": ...}`), and
`events.sweep_interval` is the backstop with the same warning when it finds unpublished entries.
Gap detection matches the bridge: a log gap is logged and the entries that are readable are published.
- **Service API.** `walgit nats` registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in)
- **Service API.** `walgit-nats` registers a NATS micro service `walgit-events` (`$SRV.PING|INFO|STATS`, built in)
with one endpoint, `<subject_prefix>.events.catch-up`. Request `{"repo": "o/r"}`, reply the catch-up report
(`repo`, `from_seq`, `head_seq`, `emitted`, `conflict`); an unknown repository is a 404 service error.
Gap, conflict and sweep counters are in the stats response. There is no HTTP listener.
Expand Down Expand Up @@ -181,19 +184,19 @@ Like the bridge it is a reader of the WAL, never a step of a push (D32). It keep
The webhook bridge has the same ambiguity from the other side: its cursor lives under the repository's prefix and
is deleted with it, so the new incarnation is published from seq 1 and the dedup key `(repo, seq, ref_name)`
collides with the old incarnation's events.
- **Config adaptation.** `walgit nats` always revalidates the manifest (`wal.freshness_ttl` is forced to zero) and
- **Config adaptation.** `walgit-nats` always revalidates the manifest (`wal.freshness_ttl` is forced to zero) and
never prefetches packs (`wal.prefetch_packs` is forced off), whatever `[wal]` says, because it never applies a
push itself and only reads manifest and log (`walgit_nats::publisher_config`).
- **Cost.** Every wake-up for a repository reads the last message on its subject (one JetStream request) before
deciding there is nothing to do, so a wiped stream is noticed without a restart.

Each behaviour above has a test against a real `nats-server -js` (`crates/walgit-nats/tests/jetstream.rs`, and
`crates/walgit-server/tests/events_nats.rs` with real git pushes); `tests/e2e.sh` runs `walgit nats` as a process.
`crates/walgit-server/tests/events_nats.rs` with real git pushes); `tests/e2e.sh` runs `walgit-nats` as a process.

| Behaviour | Test |
|---|---|
| Push becomes one message per ref with the bridge's event shape | `pushes_become_events_with_the_right_shape`, `an_atomic_multi_ref_push_is_one_message_per_ref_in_order` |
| The webhook bridge and `walgit nats` emit the same events | `the_webhook_bridge_and_the_jetstream_publisher_agree` |
| The webhook bridge and `walgit-nats` emit the same events | `the_webhook_bridge_and_the_jetstream_publisher_agree` |
| Names are encoded in the subject | `a_dotted_repository_name_is_encoded_in_the_subject` |
| The stream is the cursor; a restart resumes from it | `stream_is_the_cursor`, `a_restarted_publisher_resumes_from_the_stream` |
| A crash mid-entry finishes that entry only | `a_crash_mid_entry_finishes_that_entry_only` |
Expand Down Expand Up @@ -231,8 +234,8 @@ urls = ["nats://nats.internal:4222"]
start = "oldest"
```

Credentials are never in `walgit.toml`. `walgit nats` reads the variables the `nats` CLI reads and combines
them the way it does, so a shell where `nats` works runs `walgit nats` against the same server as the same
Credentials are never in `walgit.toml`. `walgit-nats` reads the variables the `nats` CLI reads and combines
them the way it does, so a shell where `nats` works runs `walgit-nats` against the same server as the same
user. An empty variable counts as unset.

| Variable | Effect |
Expand All @@ -247,7 +250,7 @@ user. An empty variable counts as unset.

The identity is the first of user and password, `NATS_CREDS`, `NATS_NKEY`, then `NATS_JWT`, the order `nats`
uses. A server URL written `user:password@host` or `token@host` replaces the user, password and token, as in
`nats`; servers that carry credentials must carry the same ones. `NATS_CONTEXT` and `NATS_SOCKS_PROXY` stop `walgit nats` at startup: it reads no context files and has no
`nats`; servers that carry credentials must carry the same ones. `NATS_CONTEXT` and `NATS_SOCKS_PROXY` stop `walgit-nats` at startup: it reads no context files and has no
SOCKS dialer, and ignoring either would connect differently from `nats`. `NATS_TIMEOUT` and `NATS_COLOR` shape
the CLI's own requests and output and have no effect here. Only the methods and the URLs without their
credentials are logged, never a secret.
Expand Down
6 changes: 3 additions & 3 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# walgit — a git server that is one binary in front of an object store.
#
# nix run github:<you>/walgit -- --config walgit.toml # run it
# nix build .#walgit # result/bin/{walgit,walgit-server}
# nix build .#walgit # result/bin/{walgit,walgit-server,walgit-nats}
# nix build .#image && podman load < result # an OCI image (same bytes as the Containerfile)
# nix develop # rust, protobuf, pnpm, git, just, rustfs-compatible tools
#
Expand Down Expand Up @@ -86,7 +86,7 @@
src = rustSrc;
strictDeps = true;
nativeBuildInputs = with pkgs; [ protobuf pkg-config cmake perl python3 ];
cargoExtraArgs = "-p walgit-cli --locked";
cargoExtraArgs = "-p walgit-cli -p walgit-nats --locked";
doCheck = false;
};
cargoArtifacts = craneLib.buildDepsOnly commonArgs;
Expand All @@ -101,7 +101,7 @@
# `walgit serve` shells out to git and GNU sort/comm for bounded pack planning.
nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ pkgs.makeWrapper ];
postInstall = ''
for b in walgit walgit-server; do
for b in walgit walgit-server walgit-nats; do
wrapProgram "$out/bin/$b" --prefix PATH : ${lib.makeBinPath [ pkgs.git pkgs.git-lfs pkgs.coreutils ]}
done
'';
Expand Down
4 changes: 2 additions & 2 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ sim:

# Standalone CLI/server against memory; add WALGIT_TEST_S3_ENDPOINT for the local rig.
smoke:
{{t15}} cargo build -p walgit-cli
WALGIT="$(realpath "${CARGO_TARGET_DIR:-target}/debug/walgit")" {{t15}} tests/e2e.sh
{{t15}} cargo build -p walgit-cli -p walgit-nats
WALGIT="$(realpath "${CARGO_TARGET_DIR:-target}/debug/walgit")" WALGIT_NATS="$(realpath "${CARGO_TARGET_DIR:-target}/debug/walgit-nats")" {{t15}} tests/e2e.sh

# Slow tier: #[ignore]d benches/soaks (20k-ref push, 466k-ref render, ...).
test-slow:
Expand Down
Loading