From d5666920d7292befdc2df5091abf427345764491 Mon Sep 17 00:00:00 2001 From: BuildAndDestroy Date: Wed, 9 Sep 2026 10:38:20 -0600 Subject: [PATCH 1/2] Store Commands artifacts in GridFS and ship :latest to the Reaper registry. File bytes now live in DocumentDB/Mongo GridFS so they survive pod recycles and are shared across replicas. Image build/deploy defaults to registry.reaper-ut.com/reaperc2:latest with no AWS account IDs in manifests or docs. Co-authored-by: Cursor --- Makefile | 40 +++- README.md | 134 ++++++++---- deployments/README.md | 2 +- deployments/k8s/DEPLOY.md | 9 +- deployments/k8s/reaperc2/README.md | 74 +++---- .../k8s/reaperc2/base/scripts/docdb-init.js | 14 ++ .../k8s/reaperc2/base/serviceaccount.yaml | 2 +- deployments/k8s/reaperc2/build-push-image.sh | 24 ++- deployments/k8s/reaperc2/deploy-cluster.sh | 54 ++--- .../k8s/reaperc2/examples/bedrock-irsa.md | 6 +- deployments/k8s/reaperc2/reroll.sh | 4 +- deployments/k8s/reaperc2/ship.sh | 92 +++++++++ docs/operator-guide-commands.md | 2 +- pkg/adminpanel/handlers_beacon_files.go | 2 +- pkg/dbconnections/file_artifacts.go | 126 +++++++----- .../file_artifacts_gridfs_test.go | 193 ++++++++++++++++++ pkg/dbconnections/staging_upload_expand.go | 2 +- scripts/aws-for-make.sh | 2 +- 18 files changed, 586 insertions(+), 196 deletions(-) create mode 100755 deployments/k8s/reaperc2/ship.sh create mode 100644 pkg/dbconnections/file_artifacts_gridfs_test.go diff --git a/Makefile b/Makefile index 6326597..f094bfa 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# ReaperC2 container image: build linux/amd64 + linux/arm64 and push to ECR. +# ReaperC2 container image: build linux/amd64 + linux/arm64 and push to registry.reaper-ut.com. # # Recommended on Mac (avoids go SIGSEGV under QEMU in buildx): # make build # cross-compile on host, docker only packages the image @@ -6,14 +6,14 @@ # Linux CI / full Docker compile (after `make vendor`): # make build-docker # -# Requires: docker, docker buildx, aws CLI, git. `make build` also needs Go on the host. +# Requires: docker, docker buildx, aws CLI (for ECR-backed login), git. `make build` also needs Go on the host. -AWS_ACCOUNT_ID ?= 123456789012 AWS_REGION ?= us-east-1 AWS_CLI_PROFILE ?= -ECR_REGISTRY ?= $(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com +ECR_REGISTRY ?= registry.reaper-ut.com ECR_REPOSITORY ?= reaperc2 -IMAGE_TAG ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo latest) +IMAGE_TAG ?= latest +GIT_SHA ?= $(shell git rev-parse --short HEAD 2>/dev/null) SCYTHE_GIT_REF ?= main IMAGE := $(ECR_REGISTRY)/$(ECR_REPOSITORY):$(IMAGE_TAG) @@ -30,21 +30,22 @@ DOCKER_BUILD_ARGS := \ GO_BUILD_FLAGS := -trimpath -ldflags="-s -w" -mod=vendor .PHONY: help submodule vendor build-binaries build build-docker build-amd64 build-arm64 build-local push \ - ecr-login ecr-create-repo setup-buildx + ecr-login ecr-create-repo setup-buildx ecr-alias-tags help: @echo "ReaperC2 ECR image build" @echo "" + @echo " Daily cluster update: deployments/k8s/reaperc2/ship.sh (push :latest + restart pods)" @echo " make build Host cross-compile + docker package (best on Apple Silicon)" @echo " make build-docker Full compile in Docker (needs: make vendor first)" @echo " make build-binaries Only compile bin/linux-amd64 and bin/linux-arm64/ReaperC2" - @echo " make build-amd64 Push amd64 image only" + @echo " make build-amd64 Push amd64 image only (:latest by default)" @echo " make build-arm64 Push arm64 image only" @echo " deployments/k8s/reaperc2/build-push-image.sh --arch amd64|arm64|both (wrapper)" @echo " make vendor go mod vendor (for build-docker)" @echo "" @echo "AWS: export AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY, or AWS_CLI_PROFILE=my-sso" - @echo " IMAGE=$(IMAGE)" + @echo " IMAGE=$(IMAGE) (override ECR_REGISTRY if not registry.reaper-ut.com)" submodule: git submodule update --init --recursive @@ -72,8 +73,12 @@ ecr-login: setup-buildx docker login --username AWS --password-stdin $(ECR_REGISTRY) ecr-create-repo: - @$(AWS_CMD) ecr describe-repositories --repository-names $(ECR_REPOSITORY) --region $(AWS_REGION) >/dev/null 2>&1 || \ - $(AWS_CMD) ecr create-repository --repository-name $(ECR_REPOSITORY) --region $(AWS_REGION) + @case "$(ECR_REGISTRY)" in \ + *.dkr.ecr.*.amazonaws.com) \ + $(AWS_CMD) ecr describe-repositories --repository-names $(ECR_REPOSITORY) --region $(AWS_REGION) >/dev/null 2>&1 || \ + $(AWS_CMD) ecr create-repository --repository-name $(ECR_REPOSITORY) --region $(AWS_REGION) ;; \ + *) echo "==> Skip ecr-create-repo (registry is $(ECR_REGISTRY))" ;; \ + esac # Host compile + docker pack (no go mod download / go build inside QEMU). build: build-binaries ecr-login ecr-create-repo @@ -95,6 +100,7 @@ build: build-binaries ecr-login ecr-create-repo . @echo "==> Publishing multi-arch manifest $(IMAGE)" docker buildx imagetools create -t $(IMAGE) $(IMAGE_AMD64) $(IMAGE_ARM64) + @$(MAKE) ecr-alias-tags push: build @@ -115,6 +121,18 @@ build-docker: submodule vendor ecr-login ecr-create-repo --push \ . docker buildx imagetools create -t $(IMAGE) $(IMAGE_AMD64) $(IMAGE_ARM64) + @$(MAKE) ecr-alias-tags + +# Extra ECR tags: git SHA (rollback) and :latest (cluster always pulls this). +ecr-alias-tags: + @if [ -n "$(GIT_SHA)" ] && [ "$(IMAGE_TAG)" != "$(GIT_SHA)" ]; then \ + echo "==> Also tagging $(ECR_REGISTRY)/$(ECR_REPOSITORY):$(GIT_SHA)"; \ + docker buildx imagetools create -t $(ECR_REGISTRY)/$(ECR_REPOSITORY):$(GIT_SHA) $(IMAGE); \ + fi + @if [ "$(IMAGE_TAG)" != "latest" ]; then \ + echo "==> Also tagging $(ECR_REGISTRY)/$(ECR_REPOSITORY):latest"; \ + docker buildx imagetools create -t $(ECR_REGISTRY)/$(ECR_REPOSITORY):latest $(IMAGE); \ + fi build-amd64: build-binaries ecr-login ecr-create-repo docker buildx build -f Dockerfile.pack \ @@ -124,6 +142,7 @@ build-amd64: build-binaries ecr-login ecr-create-repo $(DOCKER_BUILD_ARGS) \ --push \ . + @$(MAKE) ecr-alias-tags build-arm64: build-binaries ecr-login ecr-create-repo docker buildx build -f Dockerfile.pack \ @@ -133,6 +152,7 @@ build-arm64: build-binaries ecr-login ecr-create-repo $(DOCKER_BUILD_ARGS) \ --push \ . + @$(MAKE) ecr-alias-tags build-local: submodule vendor setup-buildx docker buildx build \ diff --git a/README.md b/README.md index eef8624..e3eaeeb 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,6 @@ The process serves **two HTTP listeners**: the **beacon API** (implants / Scythe | `BEACON_PIVOT_PROXY` | Optional default `host:port` for Scythe `--proxy` when the beacon has a **parent** (pivot). Per-beacon override: **Pivot proxy** field or `pivot_proxy` in the generate API. | | `SCYTHE_SRC` | Optional absolute path to [Scythe](https://github.com/BuildAndDestroy/Scythe) (`go.mod` + `./cmd`). If unset, ReaperC2 searches `REAPERC2_ROOT/third_party/Scythe`, then paths next to the **running binary** (covers `/root/cmd/ReaperC2` → `/root/third_party/Scythe` in Docker), then `Getwd()/third_party/Scythe`. | | `REAPERC2_ROOT` | Optional; if set, Scythe is `$REAPERC2_ROOT/third_party/Scythe`. Sample Docker Compose / K8s YAML sets `/root` for the default image; **not strictly required** if the binary path alone resolves correctly. | -| `REAPER_ARTIFACT_DIR` | Directory for staged operator uploads and files pulled from beacons via Scythe’s `download` built-in (default `./data/reaper_artifacts`). Metadata is in MongoDB collection `file_artifacts`. | | `ADMIN_SESSION_TTL_HOURS` | Server-side session lifetime (default `168`). | | `ADMIN_COOKIE_SECURE` | Set to `true` if the admin UI is only served over HTTPS (adds `Secure` on session cookies). | | `ADMIN_DISABLE` | Set to `1` to run **only** the beacon listener (no admin port). | @@ -139,7 +138,7 @@ Open `https://:8443` (or `http://` locally; `/` redirects to **Engagements |------|---------| | **Engagements** | Workspaces that scope beacons, commands, reports, topology, notes, and chat; assign operators (admins). | | **Beacons** | Generate clients, Scythe Http options, **Scythe.embedded** download (`POST /api/beacons/scythe-embedded`; Go required on server), saved profiles, kill queue. | -| **Commands** | Queue tasks; stage uploads; view pending queue, artifacts, and output history. | +| **Commands** | Queue tasks; stage uploads; view pending queue, artifacts, and output history. File bytes persist in MongoDB/DocumentDB GridFS (`reaper_artifacts`). | | **Reports** | JSON / CSV / Ghostwriter / ATT&CK Navigator layer exports. | | **Topology** | Interactive beacon graph (liveness + pivot chain). | | **Notes & ATT&CK** | Engagement notes and MITRE Navigator layer source. | @@ -166,44 +165,97 @@ With a pivot (parent beacon), the example adds `--proxy ` (from the f ### Makefile — AWS ECR (amd64 + arm64) -The root [`Makefile`](Makefile) builds a **multi-arch** image (`linux/amd64` and `linux/arm64`) with Docker **buildx** and pushes to **Amazon ECR**. Use this for EKS on x86 or Graviton nodes. +The root [`Makefile`](Makefile) builds a **multi-arch** image (`linux/amd64` and `linux/arm64`) with Docker **buildx** and pushes to **`registry.reaper-ut.com`** (override `ECR_REGISTRY` if needed). **Prerequisites** - [Docker](https://docs.docker.com/get-docker/) with the **buildx** plugin -- [AWS CLI](https://aws.amazon.com/cli/) configured (`aws sts get-caller-identity` works) -- IAM permission to push to ECR (and create the repository if it does not exist) -- [Git](https://git-scm.com/) (submodule init + default image tag from commit SHA) +- [AWS CLI](https://aws.amazon.com/cli/) v2 (`aws sts get-caller-identity` works after auth below) — used to mint the registry password +- [Git](https://git-scm.com/) (submodule init) -**Quick start** +**Authenticate to the Reaper registry** + +Two steps: set AWS CLI variables in the shell, then Docker-login to `registry.reaper-ut.com`. `make build` / `./ship.sh` also login, but you can (and should) do this yourself first. + +**1. AWS CLI variables** + +```bash +unset AWS_PROFILE + +export AWS_REGION=us-east-1 +export ECR_REPOSITORY=reaperc2 +export ECR_REGISTRY=registry.reaper-ut.com + +# Temporary creds from SSO / access portal / assume-role (keys start with ASIA): +export AWS_ACCESS_KEY_ID="ASIA..." +export AWS_SECRET_ACCESS_KEY="..." +export AWS_SESSION_TOKEN="..." +# IAM user keys (AKIA…): omit AWS_SESSION_TOKEN. + +aws sts get-caller-identity +``` + +Named profile instead of key exports: `aws sso login --profile my-sso`, then `export AWS_PROFILE=my-sso` (profile **name**, not a 12-digit account id). + +**2. Docker login** + +```bash +aws ecr get-login-password --region "${AWS_REGION}" | \ + docker login --username AWS --password-stdin "${ECR_REGISTRY}" +``` + +Expect `Login Succeeded`. Then push (`make build-amd64`) and deploy (step 3). + +The cluster pull secret is `reaperc2-myregistrykey` (`./deploy-cluster.sh ecr-secret`), with `--docker-server=registry.reaper-ut.com`. + +**3. Deploy (cluster already running)** + +Kubernetes does **not** watch `:latest`. After a successful push to `registry.reaper-ut.com/reaperc2:latest`, apply the overlay and restart pods so `imagePullPolicy: Always` pulls the new digest. + +`kubectl` must be pointed at the EKS cluster (`aws eks update-kubeconfig` if needed). DocumentDB secrets and the first `./deploy.sh all` are already done for a live cluster — do not re-run `all` or `job-docdb-user`. ```bash -# From the repo root — auth via env keys (AWS_PROFILE is ignored when these are set): -export AWS_ACCESS_KEY_ID=AKIA... -export AWS_SECRET_ACCESS_KEY=... -# export AWS_SESSION_TOKEN=... # if using temporary creds +cd deployments/k8s/reaperc2 + +# Shortcut (apply overlay + rollout restart + wait): +./reroll.sh --apply-core +# same thing: ./ship.sh --deploy-only + +# Or raw kubectl: +kubectl apply -k overlays/aws-ecr +kubectl rollout restart deployment/reaperc2-deployment -n reaperc2-ns +kubectl rollout status deployment/reaperc2-deployment -n reaperc2-ns --timeout=300s +kubectl get pods -n reaperc2-ns -o wide +``` + +Pods should become `Running` on a new age. Image on the pod should be `registry.reaper-ut.com/reaperc2:latest`. Reconnect admin with: -make build +```bash +kubectl port-forward -n reaperc2-ns deployment/reaperc2-deployment 8443:8443 ``` -Or a named profile: `make build AWS_CLI_PROFILE=your-sso-profile` (do not set `AWS_PROFILE` to the 12-digit account id; use `AWS_ACCOUNT_ID` in the Makefile for that). +Build **and** deploy in one shot (rebuilds even if you just pushed): `./ship.sh`. -That runs `git submodule update --init --recursive`, logs in to ECR, builds **amd64 and arm64 separately** (then merges into one manifest), and pushes: +**Daily loop (always `:latest`)** -`123456789012.dkr.ecr.us-east-1.amazonaws.com/reaperc2:` +```bash +# after AWS vars + docker login above +make build-amd64 # or: ./ship.sh --push-only +cd deployments/k8s/reaperc2 && ./reroll.sh --apply-core +``` -On Apple Silicon, `go mod download` inside Docker buildx often crashes (Go SIGSEGV under QEMU). **`make build`** cross-compiles on your Mac (`make vendor` + `make build-binaries`), then Docker only packages the image (`Dockerfile.pack`). Use **`make build-docker`** on native Linux CI after **`make vendor`** (full compile inside Docker with vendored modules). +On Apple Silicon, `go mod download` inside Docker buildx often crashes (Go SIGSEGV under QEMU). **`make build`** / **`./ship.sh`** cross-compile on your Mac (`make vendor` + `make build-binaries`), then Docker only packages the image (`Dockerfile.pack`). Use **`make build-docker`** on native Linux CI after **`make vendor`**. -**Release tag** +**Pin a release tag** (still also aliases `:latest` and the git SHA): ```bash make build IMAGE_TAG=v1.0.0 ``` -**Other accounts or regions** +**Other registry hosts** ```bash -make build AWS_ACCOUNT_ID=123456789012 AWS_REGION=us-west-2 ECR_REPOSITORY=reaperc2 +make build ECR_REGISTRY=registry.example.com AWS_REGION=us-east-1 ``` **Makefile targets** @@ -211,33 +263,36 @@ make build AWS_ACCOUNT_ID=123456789012 AWS_REGION=us-west-2 ECR_REPOSITORY=reape | Target | Description | |--------|-------------| | `make help` | List targets and current `IMAGE` | -| `make build` | Host cross-compile + multi-arch ECR push (recommended on Mac) | +| `make build` | Host cross-compile + multi-arch push to `ECR_REGISTRY` (recommended on Mac) | | `make build-docker` | Full Docker build with vendored modules (Linux CI) | | `make build-binaries` | Only `bin/linux-amd64` and `bin/linux-arm64/ReaperC2` | | `make vendor` | `go mod vendor` (required before `build-docker`) | | `make push` | Same as `make build` | -| `make build-amd64` | Push only `...:$(IMAGE_TAG)-amd64` | +| `make build-amd64` | Push only `...:$(IMAGE_TAG)-amd64` (default tag `latest`) | | `make build-arm64` | Push only `...:$(IMAGE_TAG)-arm64` | -| `make build-local` | Build `reaperc2:local` for your machine (`--load`, no ECR) | -| `make ecr-login` | ECR docker login only | -| `make ecr-create-repo` | Create the ECR repository if missing | +| `make build-local` | Build `reaperc2:local` for your machine (`--load`, no registry) | +| `make ecr-login` | `get-login-password` → `docker login $(ECR_REGISTRY)` | +| `make ecr-create-repo` | Create the AWS ECR repository if `ECR_REGISTRY` is an `*.dkr.ecr.*` host | +| [`./ship.sh`](deployments/k8s/reaperc2/ship.sh) | Push `:latest` (amd64) + apply-core + rollout | **Variables** (override on the command line or in the environment) | Variable | Default | Purpose | |----------|---------|---------| -| `AWS_ACCOUNT_ID` | `123456789012` | ECR registry account (override with your account) | -| `AWS_REGION` | `us-east-1` | ECR region | -| `ECR_REPOSITORY` | `reaperc2` | Repository name | -| `IMAGE_TAG` | `git rev-parse --short HEAD` | Image tag (`latest` if not in a git repo) | +| `ECR_REGISTRY` | `registry.reaper-ut.com` | Docker login and push host | +| `ECR_REPOSITORY` | `reaperc2` | Repository name (`registry.reaper-ut.com/reaperc2:latest`) | +| `IMAGE_TAG` | `latest` | Image tag; every push also aliases the git SHA, and aliases `:latest` if you override this | +| `AWS_REGION` | `us-east-1` | Region for `aws ecr get-login-password` | | `SCYTHE_GIT_REF` | `main` | Branch/tag when the Dockerfile must clone Scythe (submodule preferred) | -| `AWS_CLI_PROFILE` | (unset) | Optional `aws --profile` for ECR login; env `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` take precedence | +| `AWS_ACCESS_KEY_ID` | (unset) | Access key (`ASIA…` STS or `AKIA…` IAM user) | +| `AWS_SECRET_ACCESS_KEY` | (unset) | Secret key paired with `AWS_ACCESS_KEY_ID` | +| `AWS_SESSION_TOKEN` | (unset) | **Required** for temporary/`ASIA` creds; omit for IAM user keys | +| `AWS_CLI_PROFILE` | (unset) | Named `aws --profile` for registry login; takes precedence over env keys | +| `AWS_PROFILE` | (unset) | Shell profile if no `AWS_CLI_PROFILE` and no env keys. Must be a profile **name**, never an account id. Ignored when `AWS_ACCESS_KEY_ID` is set. | **Deploy to EKS after push** -1. Set the image in [`deployments/k8s/reaperc2/base/deployment.yaml`](deployments/k8s/reaperc2/base/deployment.yaml) to the tag you pushed, e.g. `123456789012.dkr.ecr.us-east-1.amazonaws.com/reaperc2:v1.0.0` (use your `AWS_ACCOUNT_ID`). -2. Follow [`deployments/k8s/reaperc2/README.md`](deployments/k8s/reaperc2/README.md#quick-install-script): run [`deployments/k8s/reaperc2/deploy.sh`](deployments/k8s/reaperc2/deploy.sh) `all` (or [`deploy-cluster.sh`](deployments/k8s/reaperc2/deploy-cluster.sh) `all`), optionally with **`--no-egress`** / **`--with-egress`**; or manually `fetch-ca`, secrets, `kubectl apply -k deployments/k8s/AWS` (legacy shim) or `kubectl apply -k deployments/k8s/reaperc2/overlays/aws-ecr`, DocumentDB Jobs, then `apply-ingress` when Traefik/cert-manager are ready. After you edit `base/deployment.yaml` (new image tag), use [`reroll.sh --apply-core`](deployments/k8s/reaperc2/reroll.sh) or **`./deploy.sh apply-core`** so the cluster picks up the manifest; **`./reroll.sh`** alone only restarts pods on the current spec. -3. Roll out: `kubectl rollout restart deployment/reaperc2-deployment -n reaperc2-ns` +First install: follow [`deployments/k8s/reaperc2/README.md`](deployments/k8s/reaperc2/README.md#quick-install-script) (`./deploy.sh all`, DocumentDB Jobs, `apply-ingress`). After that, **do not edit the image tag** — keep `…/reaperc2:latest` and run [`./ship.sh`](deployments/k8s/reaperc2/ship.sh). ### Docker build (single arch, any registry) @@ -332,24 +387,17 @@ Configure **`BEACON_PUBLIC_BASE_URL`** (and/or each beacon’s **Beacon C2 base | Path | Use when | |------|----------| -| [`deployments/k8s/`](deployments/k8s/) | EKS or **k3s** + **DocumentDB** + Traefik/cert-manager ([`deploy-cluster.sh`](deployments/k8s/reaperc2/deploy-cluster.sh), [`deploy.sh`](deployments/k8s/reaperc2/deploy.sh), [`reroll.sh`](deployments/k8s/reaperc2/reroll.sh); quick index [`deployments/k8s/DEPLOY.md`](deployments/k8s/DEPLOY.md); `kubectl apply -k deployments/k8s/AWS` still works as **aws-ecr** shim) | +| [`deployments/k8s/`](deployments/k8s/) | EKS or **k3s** + **DocumentDB** + Traefik/cert-manager ([`ship.sh`](deployments/k8s/reaperc2/ship.sh), [`deploy-cluster.sh`](deployments/k8s/reaperc2/deploy-cluster.sh), [`deploy.sh`](deployments/k8s/reaperc2/deploy.sh), [`reroll.sh`](deployments/k8s/reaperc2/reroll.sh); quick index [`deployments/k8s/DEPLOY.md`](deployments/k8s/DEPLOY.md); `kubectl apply -k deployments/k8s/AWS` still works as **aws-ecr** shim) | | [`deployments/k8s/OnPrem/`](deployments/k8s/OnPrem/) | In-cluster MongoDB | | [`deployments/k8s/full-deployment.yaml`](deployments/k8s/full-deployment.yaml) | Sample all-in-one with in-cluster Mongo | **AWS deploy (summary)** ```bash -make build # or: make build IMAGE_TAG=v1.0.0 -# Edit deployments/k8s/reaperc2/base/deployment.yaml (ECR image), ingress hostnames, examples/documentdb-secret.yaml - cd deployments/k8s/reaperc2 -./fetch-docdb-ca-bundle.sh -# Prefer: ./deploy-cluster.sh all (then job-docdb-user / job-docdb-init / apply-ingress) -kubectl apply -f namespace.yaml -f examples/documentdb-secret.yaml -# ECR pull secret + docdb jobs — see AWS README -kubectl apply -k . -kubectl apply -f docdb-init-job.yaml -kubectl wait -n reaperc2-ns job/docdb-init --for=condition=complete --timeout=120s +# First time: ./deploy.sh all then job-docdb-user / job-docdb-init / apply-ingress +# Every image update: +./ship.sh ``` * Point **Ingress / IngressRoute** only at Service port **8080** (beacon). diff --git a/deployments/README.md b/deployments/README.md index 78d68c2..7855e70 100644 --- a/deployments/README.md +++ b/deployments/README.md @@ -37,7 +37,7 @@ REGISTRY_PASSWORD='your-password-here' | `REAPERC2_NAMESPACE` | Default `reaperc2-ns` | | `KUBECONFIG` | Optional kubeconfig path | -**DocumentDB / ECR builds** use [`k8s/reaperc2/build-push-image.sh`](k8s/reaperc2/build-push-image.sh) and the Makefile (`AWS_ACCOUNT_ID`, `AWS_REGION`, etc.) — not `config.env`. +**DocumentDB / registry builds** use [`k8s/reaperc2/build-push-image.sh`](k8s/reaperc2/build-push-image.sh) and the Makefile (`ECR_REGISTRY=registry.reaper-ut.com`, `AWS_REGION`, etc.) — not `config.env`. --- diff --git a/deployments/k8s/DEPLOY.md b/deployments/k8s/DEPLOY.md index 522ebbf..c1dbc8d 100644 --- a/deployments/k8s/DEPLOY.md +++ b/deployments/k8s/DEPLOY.md @@ -24,11 +24,10 @@ Supported path: **DocumentDB**, Traefik, cert-manager, Bedrock Operator AI. ```bash cd deployments/k8s/reaperc2 -chmod +x deploy.sh reroll.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh +chmod +x deploy.sh reroll.sh ship.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh # AWS: export AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN; unset AWS_PROFILE -# export AWS_ACCOUNT_ID=… AWS_REGION=us-east-1 (match base/deployment.yaml ECR host) -# Build and push: ./build-push-image.sh --arch amd64 # or arm64 / both -# Copy and edit examples/*.local.yaml, base/deployment.yaml image, optional ../operator-ai.local.yaml +# Build/push :latest and restart: ./ship.sh +# First install only: copy examples/*.local.yaml, optional ../operator-ai.local.yaml ./deploy.sh check-local ./deploy.sh all # or: ./deploy-cluster.sh all ./deploy.sh job-docdb-user @@ -38,7 +37,7 @@ chmod +x deploy.sh reroll.sh build-push-image.sh deploy-cluster.sh base/fetch-do **Egress lockdown (optional):** copy `reaperc2/examples/networkpolicy-egress-restricted.yaml` → `networkpolicy-egress-restricted.local.yaml`, edit DocumentDB CIDR, then `./deploy.sh --with-egress all`. Requires a CNI that enforces NetworkPolicy. -**After you change secrets, manifests, or the image tag:** `./reroll.sh --apply-core` if `base/deployment.yaml` (or overlay) changed in git; `./reroll.sh` alone only restarts pods without applying YAML from disk. Add `--apply-secrets` / `--refresh-ecr` as needed. +**After you change secrets or manifests:** `./reroll.sh --apply-core`. **After a new image:** `./ship.sh` (push `:latest` + restart). `./reroll.sh` alone restarts pods without pushing. **k3s:** `REAPER_CLUSTER=k3s ./deploy.sh all` (same scripts). diff --git a/deployments/k8s/reaperc2/README.md b/deployments/k8s/reaperc2/README.md index d533578..e07773d 100644 --- a/deployments/k8s/reaperc2/README.md +++ b/deployments/k8s/reaperc2/README.md @@ -23,7 +23,7 @@ These caused most deploy pain — avoid them up front: | **`docdb-init` Job errors** | Run `./deploy-cluster.sh apply-core` first (refreshes SCRAM-SHA-1 scripts), then re-apply the Job. | | **App still uses wrong auth DB** | `base/deployment.yaml` reads `auth_source` from the secret. After editing the secret: `kubectl apply -f base/deployment.yaml` and rollout restart. | -ReaperC2 only needs DocumentDB for data — no Kubernetes PVC for the app. Operator AI includes **in-cluster Ollama** by default ([`../ollama.yaml`](../ollama.yaml); applied by `./deploy-cluster.sh apply-ollama` / `all`) plus optional Bedrock and other cloud providers (see [Bedrock credentials](#bedrock-credentials-rotation)). Skip Ollama with `SKIP_OLLAMA=1` if the node cannot host models. +ReaperC2 only needs DocumentDB for data — no Kubernetes PVC for the app. Staged uploads and beacon downloads on **Commands** are stored in DocumentDB (metadata in `file_artifacts`, bytes in GridFS bucket `reaper_artifacts`) so they survive pod recycles and are visible to all replicas. Operator AI includes **in-cluster Ollama** by default ([`../ollama.yaml`](../ollama.yaml); applied by `./deploy-cluster.sh apply-ollama` / `all`) plus optional Bedrock and other cloud providers (see [Bedrock credentials](#bedrock-credentials-rotation)). Skip Ollama with `SKIP_OLLAMA=1` if the node cannot host models. ## Quick install (script) @@ -31,7 +31,7 @@ ReaperC2 only needs DocumentDB for data — no Kubernetes PVC for the app. Opera ```bash cd deployments/k8s/reaperc2 -chmod +x deploy.sh reroll.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh +chmod +x deploy.sh reroll.sh ship.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh # EKS / ECR (default REAPER_CLUSTER=aws): ./deploy-cluster.sh check-local ./deploy-cluster.sh help @@ -53,13 +53,14 @@ REAPER_CLUSTER=k3s ./deploy-cluster.sh apply-ingress | Script | Purpose | |--------|---------| | [`deploy.sh`](deploy.sh) | Wrapper around [`deploy-cluster.sh`](deploy-cluster.sh). Use **`--no-egress`** (default path: do nothing extra) to **remove** `NetworkPolicy/reaperc2-egress-restricted` before `all` / `apply-core` so pod egress is open. Use **`--with-egress`** to **apply** [`examples/networkpolicy-egress-restricted.local.yaml`](examples/networkpolicy-egress-restricted.yaml) after `all` / `apply-core` (you must copy the template to `.local.yaml` and fix **DocumentDB CIDR**). **`teardown`** always deletes that NetworkPolicy after the main teardown. | -| [`reroll.sh`](reroll.sh) | **`./reroll.sh`** — `rollout restart` only (cluster keeps current Deployment **image** and YAML). **`./reroll.sh --apply-core`** — `kubectl apply -k` overlay first (use after editing `base/deployment.yaml` / manifests in git), then restart. **`--apply-secrets`** / **`--refresh-ecr`** — same as before. | +| [`reroll.sh`](reroll.sh) | **`./reroll.sh`** — `rollout restart` only (cluster keeps current Deployment **image** and YAML). **`./reroll.sh --apply-core`** — `kubectl apply -k` overlay first, then restart. **`--apply-secrets`** / **`--refresh-ecr`** — same as before. | +| [`ship.sh`](ship.sh) | **Daily image update:** push ECR **`:latest`** (amd64) then apply-core + rollout. Prefer this over editing image tags. | | [`build-push-image.sh`](build-push-image.sh) | **`./build-push-image.sh --arch amd64|arm64|both`** — export AWS creds first (see [Build and push](#build-and-push)), then runs `make` from repo root. | **Directions (short):** -1. `cd deployments/k8s/reaperc2` and `chmod +x deploy.sh reroll.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh`. -2. Copy and edit the three secret templates + `base/deployment.yaml` image + optional `../operator-ai.local.yaml` (see [Run from scratch](#run-from-scratch-checklist)). +1. `cd deployments/k8s/reaperc2` and `chmod +x deploy.sh reroll.sh ship.sh build-push-image.sh deploy-cluster.sh base/fetch-docdb-ca-bundle.sh`. +2. Copy and edit the three secret templates + optional `../operator-ai.local.yaml` (see [Run from scratch](#run-from-scratch-checklist)). Keep `base/deployment.yaml` on **`:latest`**. 3. **Deploy without egress restrictions** (typical first bring-up): `./deploy.sh check-local` then **`./deploy.sh all`** or **`./deploy.sh --no-egress all`**. Then `./deploy.sh job-docdb-user`, optional `./deploy.sh job-docdb-init`, then **`./deploy.sh apply-ingress`** when Traefik + cert-manager are ready. 4. **Deploy with egress restrictions:** copy [`examples/networkpolicy-egress-restricted.yaml`](examples/networkpolicy-egress-restricted.yaml) → `examples/networkpolicy-egress-restricted.local.yaml`, set the **DocumentDB** `ipBlock` CIDR to your VPC (or tighter), add any extra egress rules you need, then **`./deploy.sh --with-egress all`** (or `--with-egress apply-core`). **Only works if your CNI enforces NetworkPolicy** — default EKS/VPC CNI may not; use Calico/Cilium or [AWS VPC CNI network policy mode](https://docs.aws.amazon.com/eks/latest/userguide/network-policies.html) as appropriate. 5. **Reroll** after image or secret changes: **`./reroll.sh --apply-core`** if you changed `base/deployment.yaml` (or other kustomize files) in git — plain **`./reroll.sh`** only restarts pods on the **already-applied** spec. Combine with **`--apply-secrets`** / **`--refresh-ecr`** as needed. @@ -80,7 +81,7 @@ Use the **same** Traefik IngressClass (`traefik`), cert-manager, `ingress.yaml`, ## Run from scratch (checklist) -From the **repo root**, after [`Build and push`](#build-and-push) (export AWS creds, set `AWS_ACCOUNT_ID` / `AWS_REGION`, then `./build-push-image.sh --arch amd64` from `deployments/k8s/reaperc2`): +From the **repo root**, after [`Build and push`](#build-and-push) (export AWS creds, `ECR_REGISTRY=registry.reaper-ut.com`, then `./build-push-image.sh --arch amd64` from `deployments/k8s/reaperc2`): **0. Edit local files (do not commit secrets)** @@ -96,7 +97,7 @@ Edit: - `examples/documentdb-secret.local.yaml` — host, `username`, `password`, `database`, **`auth_source` (same as `database`)** - `examples/documentdb-admin-secret.local.yaml` — DocumentDB **master** user (init Job only) - `examples/admin-bootstrap-secret.local.yaml` — first **admin UI** login (only when `operators` collection is empty) -- `base/deployment.yaml` — container `image:` (ECR on aws profile; any registry on k3s) +- `base/deployment.yaml` — image stays **`registry.reaper-ut.com/reaperc2:latest`**; do not pin account-specific ECR hosts. - `ingress.yaml` / `ingressroute.yaml` — beacon hostname **1. TLS CA bundle** (once per clone; gitignored) @@ -187,77 +188,80 @@ Skip the user Job only if your **infra repo** already created the user with exac ## Prerequisites - `kubectl` pointed at your cluster (`aws eks update-kubeconfig ...` for EKS, or your k3s kubeconfig) -- ReaperC2 image built and pushed to ECR (see [Build and push](#build-and-push); export `AWS_*` creds and set `AWS_ACCOUNT_ID` to match `base/deployment.yaml`) +- ReaperC2 image built and pushed to `registry.reaper-ut.com` (see [Build and push](#build-and-push)) - DocumentDB cluster endpoint and application DB user (`api_user` / `api_db` or your naming) - Traefik installed with an `IngressClass` named `traefik` (adjust manifests if yours differs) ## Build and push -Build the image from this directory with [`build-push-image.sh`](build-push-image.sh), or from the repo root with `make`. Both use the same Makefile targets and ECR settings. +Build the image from this directory with [`build-push-image.sh`](build-push-image.sh), or from the repo root with `make`. Both use the same Makefile targets. Default registry is **`registry.reaper-ut.com`**. ### AWS credentials (export) Most operators authenticate with **temporary credentials** exported into the shell (SSO login, `aws sts assume-role`, or the AWS access portal). Export all three variables when your keys start with `ASIA` (STS): ```bash -# From SSO / access portal / assume-role — paste the three export lines, then: -unset AWS_PROFILE # env keys take precedence; avoid a stale profile pointing at the wrong account +unset AWS_PROFILE -export AWS_ACCOUNT_ID=235360402887 # must match base/deployment.yaml ECR host (not the Makefile placeholder) -export AWS_REGION=us-east-1 # must match the region in your ECR URI +export AWS_REGION=us-east-1 +export ECR_REGISTRY=registry.reaper-ut.com +export ECR_REPOSITORY=reaperc2 export AWS_ACCESS_KEY_ID="ASIA..." export AWS_SECRET_ACCESS_KEY="..." export AWS_SESSION_TOKEN="..." # required for temporary creds; omit only for long-lived IAM user keys (AKIA…) -``` - -Verify the account matches before pushing: -```bash aws sts get-caller-identity -# Account should equal AWS_ACCOUNT_ID and the numeric prefix in base/deployment.yaml image: -# 235360402887.dkr.ecr.us-east-1.amazonaws.com/reaperc2:... + +aws ecr get-login-password --region "${AWS_REGION}" | \ + docker login --username AWS --password-stdin "${ECR_REGISTRY}" ``` **Alternative — named profile** (no manual export): `make build AWS_CLI_PROFILE=my-sso` or `./build-push-image.sh --arch amd64 AWS_CLI_PROFILE=my-sso`. See [`scripts/aws-for-make.sh`](../../../scripts/aws-for-make.sh) for auth precedence. -### Build and push to ECR +### Build and push -From **`deployments/k8s/reaperc2`** (recommended): +**Daily (recommended):** push `:latest` and restart pods so `imagePullPolicy: Always` pulls the new digest. ```bash cd deployments/k8s/reaperc2 -chmod +x build-push-image.sh -./build-push-image.sh --arch amd64 # EKS x86_64 nodes -# ./build-push-image.sh --arch arm64 # Graviton / ARM k3s -# ./build-push-image.sh --arch both # multi-arch manifest +chmod +x ship.sh build-push-image.sh +./ship.sh # amd64, tag latest, apply-core + rollout +# ./ship.sh --arch both +# ./ship.sh --push-only +# ./ship.sh --deploy-only # already pushed ``` -From the **repo root** (same result): +Rollback: the same push also tags the git short SHA (`registry.reaper-ut.com/reaperc2:`). + +Lower-level (same Makefile): ```bash -make build-amd64 -# Or: make build (amd64 + arm64 manifest) -# Pin a tag: IMAGE_TAG=v1.0.0 make build-amd64 +./build-push-image.sh --arch amd64 # EKS x86_64 nodes +# ./build-push-image.sh --arch arm64 # Graviton / ARM k3s +# ./build-push-image.sh --arch both # multi-arch manifest +# From repo root: make build-amd64 # IMAGE_TAG defaults to latest +# Pin a release: IMAGE_TAG=v1.0.0 make build-amd64 # also aliases :latest ``` -Requires Docker with **buildx**, the **AWS CLI**, and ECR permissions (`ecr:GetAuthorizationToken`, `ecr:BatchCheckLayerAvailability`, `ecr:PutImage`, etc.). The Makefile runs `git submodule update --init --recursive` before build so Scythe matches this repo. +Requires Docker with **buildx**, the **AWS CLI**, and permission to `ecr:GetAuthorizationToken` (login password). The Makefile runs `git submodule update --init --recursive` before build so Scythe matches this repo. | Target / script | Purpose | |-----------------|---------| +| `./ship.sh` | Push `:latest` (default amd64) + apply overlay + rollout | | `./build-push-image.sh --arch amd64` | Push `...:$(IMAGE_TAG)-amd64` and tag manifest `:$(IMAGE_TAG)` (amd64 only) | | `./build-push-image.sh --arch arm64` | Push arm64 only | | `./build-push-image.sh --arch both` | Multi-arch manifest at `:$(IMAGE_TAG)` | -| `make build-local` | Load single-arch image `reaperc2:local` locally (no ECR push) | +| `make build-local` | Load single-arch image `reaperc2:local` locally (no registry push) | -Variables: `AWS_ACCOUNT_ID`, `AWS_REGION`, `ECR_REPOSITORY` (default `reaperc2`), `IMAGE_TAG` (default short git SHA), `SCYTHE_GIT_REF`. Run `make help` for defaults. +Variables: `ECR_REGISTRY` (default `registry.reaper-ut.com`), `ECR_REPOSITORY` (default `reaperc2`), `IMAGE_TAG` (default **`latest`**), `AWS_REGION`, `SCYTHE_GIT_REF`. Run `make help` for defaults. -After a successful push, confirm `base/deployment.yaml` `image:` matches the URI you built (account, region, repository, tag). Then roll out: `./reroll.sh --apply-core`. +After a successful push, pods only pull when they are **recreated**. `./ship.sh` does that. `./reroll.sh` restarts without pushing. | Build symptom | Likely cause | |---------------|--------------| -| `403 Forbidden` pushing to ECR | `AWS_ACCOUNT_ID` still at Makefile default `123456789012`, or exported creds are for a different account than the ECR registry. Run `aws sts get-caller-identity` and set `AWS_ACCOUNT_ID` to that account (or fix `deployment.yaml`). | -| `Login Succeeded` then push fails | Same as above — login and push target different accounts/regions. | +| `403 Forbidden` pushing | Docker not logged in to `ECR_REGISTRY`, or creds cannot push to that registry. Re-run the `get-login-password \| docker login` command. | +| `Login Succeeded` then push fails | Login host and `ECR_REGISTRY` used by `make` differ. Export `ECR_REGISTRY=registry.reaper-ut.com`. | | `error: AWS_PROFILE=… looks like an account id` | You set `AWS_PROFILE` to a 12-digit account id. `unset AWS_PROFILE` and use the three `export` lines, or set `AWS_PROFILE` to a **profile name** from `~/.aws/config`. | ## Configure before apply diff --git a/deployments/k8s/reaperc2/base/scripts/docdb-init.js b/deployments/k8s/reaperc2/base/scripts/docdb-init.js index f63f824..a0ef1b8 100644 --- a/deployments/k8s/reaperc2/base/scripts/docdb-init.js +++ b/deployments/k8s/reaperc2/base/scripts/docdb-init.js @@ -68,6 +68,20 @@ ensureIndex( { name: "idx_file_artifacts_client_created" } ); +// GridFS bucket reaper_artifacts (driver also creates these on first upload). +ensureCollection("reaper_artifacts.files"); +ensureIndex( + "reaper_artifacts.files", + { filename: 1, uploadDate: 1 }, + { name: "idx_reaper_artifacts_files_filename_uploaddate" } +); +ensureCollection("reaper_artifacts.chunks"); +ensureIndex( + "reaper_artifacts.chunks", + { files_id: 1, n: 1 }, + { unique: true, name: "idx_reaper_artifacts_chunks_files_id_n" } +); + ensureCollection("beacon_profiles"); ensureIndex("beacon_profiles", { created_at: -1 }, { name: "idx_beacon_profiles_created_at" }); diff --git a/deployments/k8s/reaperc2/base/serviceaccount.yaml b/deployments/k8s/reaperc2/base/serviceaccount.yaml index 760cc4b..26ec47c 100644 --- a/deployments/k8s/reaperc2/base/serviceaccount.yaml +++ b/deployments/k8s/reaperc2/base/serviceaccount.yaml @@ -6,4 +6,4 @@ metadata: namespace: reaperc2-ns # After creating the Bedrock IAM role, uncomment and set your role ARN: # annotations: - # eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/reaperc2-bedrock + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/reaperc2-bedrock diff --git a/deployments/k8s/reaperc2/build-push-image.sh b/deployments/k8s/reaperc2/build-push-image.sh index 10edfd4..7358627 100755 --- a/deployments/k8s/reaperc2/build-push-image.sh +++ b/deployments/k8s/reaperc2/build-push-image.sh @@ -7,7 +7,8 @@ # ./build-push-image.sh --arch both # amd64 + arm64 + multi-arch manifest (default make build) # # Requires: Docker with buildx, Go, AWS CLI (same as `make build` / `make help`). -# Set AWS_ACCOUNT_ID, AWS_REGION, ECR_REPOSITORY, IMAGE_TAG, AWS_CLI_PROFILE, SCYTHE_GIT_REF as for make. +# Set ECR_REGISTRY, AWS_REGION, ECR_REPOSITORY, IMAGE_TAG, AWS_CLI_PROFILE, SCYTHE_GIT_REF as for make. +# IMAGE_TAG defaults to latest. Daily cluster update: ./ship.sh (push :latest + restart). # # Note: make build-amd64 / build-arm64 still run `build-binaries` (compiles linux/amd64 and linux/arm64 # Go binaries on the host); only the selected image is packaged and pushed. @@ -21,7 +22,7 @@ usage() { cat <<'EOF' Usage: ./build-push-image.sh --arch [make-vars...] - Runs `make` from the ReaperC2 repo root to compile and push the image to ECR. + Runs `make` from the ReaperC2 repo root to compile and push the image (default registry.reaper-ut.com). --arch amd64 Same as `make build-amd64` — push :TAG-amd64 and set the :TAG manifest to amd64-only. --arch arm64 Same as `make build-arm64` — push :TAG-arm64 and set the :TAG manifest to arm64-only. @@ -30,10 +31,10 @@ Usage: ./build-push-image.sh --arch [make-vars...] Aliases for --arch: x86_64→amd64, aarch64|arm→arm64, multi|all→both. Environment (optional, passed to make): - AWS_ACCOUNT_ID AWS account (default in Makefile if unset) - AWS_REGION e.g. us-east-1 + ECR_REGISTRY default registry.reaper-ut.com + AWS_REGION e.g. us-east-1 (for get-login-password) ECR_REPOSITORY default reaperc2 - IMAGE_TAG default short git SHA + IMAGE_TAG default latest (also aliases git SHA; :latest is always pushed) AWS_CLI_PROFILE SSO / named profile (see scripts/aws-for-make.sh) SCYTHE_GIT_REF Scythe submodule ref for the image build @@ -41,12 +42,14 @@ Environment (optional, passed to make): Examples: cd deployments/k8s/reaperc2 - chmod +x build-push-image.sh + chmod +x build-push-image.sh ship.sh + ./ship.sh # push :latest (amd64) and restart pods ./build-push-image.sh --arch amd64 ./build-push-image.sh --arch arm64 AWS_CLI_PROFILE=my-sso IMAGE_TAG=v1.2.3 REAPER_IMAGE_ARCH=both ./build-push-image.sh -After a successful push, set base/deployment.yaml image: to the printed ECR URI (or use IMAGE_TAG with your registry path). +After a successful push, Deployment image: .../reaperc2:latest (imagePullPolicy: Always). +Restart pods to pull: ./ship.sh (includes restart) or ./reroll.sh --apply-core EOF } @@ -90,6 +93,7 @@ else fi echo "" -echo "Update your Deployment image to the tag you built (see make output above). Typical form:" -echo " \${AWS_ACCOUNT_ID}.dkr.ecr.\${AWS_REGION}.amazonaws.com/reaperc2:\${IMAGE_TAG}" -echo "For this tree, IMAGE_TAG defaults to: $(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo '')" +echo "Pushed ${ECR_REGISTRY:-registry.reaper-ut.com}/reaperc2:${IMAGE_TAG:-latest}" +echo "Cluster spec uses registry.reaper-ut.com/reaperc2:latest (imagePullPolicy: Always). Restart pods with:" +echo " ./ship.sh # this push + apply-core + rollout" +echo " ./reroll.sh --apply-core" diff --git a/deployments/k8s/reaperc2/deploy-cluster.sh b/deployments/k8s/reaperc2/deploy-cluster.sh index 7d29199..1e4e1af 100755 --- a/deployments/k8s/reaperc2/deploy-cluster.sh +++ b/deployments/k8s/reaperc2/deploy-cluster.sh @@ -33,7 +33,7 @@ usage() { cat <<'EOF' Usage: ./deploy-cluster.sh [args] -Also see ./deploy.sh (optional --with-egress / --no-egress) and ./reroll.sh (rollout + optional secrets/ECR refresh). +Also see ./deploy.sh, ./reroll.sh, and ./ship.sh (push ECR :latest + restart). Commands: help Show this help. @@ -54,10 +54,10 @@ Commands: teardown Remove app from cluster (same as README Teardown): kustomize stack, ingress, jobs, sample secret apply, ollama stack. Does not delete the namespace or DocumentDB. Environment: - REAPER_CLUSTER aws (default) or k3s — picks Kustomize overlay and ECR checks. + REAPER_CLUSTER aws (default) or k3s — picks Kustomize overlay. REAPER_NS Namespace (default: reaperc2-ns) - REAPER_ECR_ACCOUNT AWS account id for ECR (optional if deployment.yaml image is a real *.dkr.ecr..amazonaws.com URI) - AWS_REGION ECR region for aws ecr get-login-password (optional if derivable from deployment image) + ECR_REGISTRY Registry host for ecr-secret (default: host of base/deployment.yaml image, usually registry.reaper-ut.com) + AWS_REGION Region for aws ecr get-login-password (default: us-east-1) SKIP_ECR_SECRET Set to 1 to skip ecr-secret in "all" (aws). On k3s, ecr-secret is skipped unless REAPER_ECR_SECRET=1. REAPER_ECR_SECRET Set to 1 on k3s to run ecr-secret in "all" (same as aws flow). SKIP_OLLAMA Set to 1 to skip in-cluster Ollama on apply-ollama / all (Operator AI can still use cloud providers). @@ -146,24 +146,11 @@ cmd_apply_secrets() { fi } -ecr_account_from_deployment() { +registry_host_from_deployment() { local img img="$(grep -E '^\s+image:\s' "$BASE/deployment.yaml" | head -1 | awk '{print $2}')" - if [[ "$img" =~ ^([0-9]{12})\.dkr\.ecr\.([a-z0-9-]+)\.amazonaws\.com/ ]]; then - echo "${BASH_REMATCH[1]}" - return - fi - return 1 -} - -ecr_region_from_deployment() { - local img - img="$(grep -E '^\s+image:\s' "$BASE/deployment.yaml" | head -1 | awk '{print $2}')" - if [[ "$img" =~ \.dkr\.ecr\.([a-z0-9-]+)\.amazonaws\.com ]]; then - echo "${BASH_REMATCH[1]}" - return - fi - return 1 + [[ -n "$img" ]] || return 1 + echo "${img%%/*}" } require_deploy_prereqs() { @@ -201,9 +188,9 @@ EOM [[ -x "$BASE/fetch-docdb-ca-bundle.sh" ]] || die "run: chmod +x $BASE/fetch-docdb-ca-bundle.sh" if [[ "${REAPER_CLUSTER}" == "aws" ]]; then - local acct - acct="$(ecr_account_from_deployment || true)" - [[ -n "$acct" && "$acct" != "123456789012" ]] || die "aws profile: base/deployment.yaml image must be a real ECR URI (replace placeholder account 123456789012 with your AWS account id), or use REAPER_CLUSTER=k3s for non-ECR images." + local img + img="$(grep -E '^\s+image:\s' "$BASE/deployment.yaml" | head -1 | awk '{print $2}')" + [[ -n "$img" ]] || die "aws profile: base/deployment.yaml is missing image:" fi } @@ -214,22 +201,19 @@ cmd_check_local() { cmd_ecr_secret() { require_deploy_prereqs - [[ "${REAPER_CLUSTER}" == "aws" || "${REAPER_ECR_SECRET:-0}" == "1" ]] || die "ecr-secret is for ECR pulls. Use REAPER_CLUSTER=aws, or REAPER_ECR_SECRET=1 with k3s." - local acct region - acct="${REAPER_ECR_ACCOUNT:-}" - region="${AWS_REGION:-}" - if [[ -z "$acct" ]]; then - acct="$(ecr_account_from_deployment || true)" - fi - [[ -n "$acct" && "$acct" != "123456789012" ]] || die "Set REAPER_ECR_ACCOUNT or fix base/deployment.yaml image to your real ECR URI (not placeholder 123456789012)." - if [[ -z "$region" ]]; then - region="$(ecr_region_from_deployment || echo us-east-1)" + [[ "${REAPER_CLUSTER}" == "aws" || "${REAPER_ECR_SECRET:-0}" == "1" ]] || die "ecr-secret is for registry pulls. Use REAPER_CLUSTER=aws, or REAPER_ECR_SECRET=1 with k3s." + local server region + server="${ECR_REGISTRY:-}" + if [[ -z "$server" ]]; then + server="$(registry_host_from_deployment || true)" fi + [[ -n "$server" ]] || die "set ECR_REGISTRY or put a registry host on base/deployment.yaml image:" + region="${AWS_REGION:-us-east-1}" command -v aws >/dev/null || die "aws CLI required for ecr-secret" - info "ECR docker-registry secret for ${acct}.dkr.ecr.${region}.amazonaws.com" + info "docker-registry secret for ${server}" "$KUBECTL" create secret docker-registry reaperc2-myregistrykey \ --namespace="$REAPER_NS" \ - --docker-server="${acct}.dkr.ecr.${region}.amazonaws.com" \ + --docker-server="${server}" \ --docker-username=AWS \ --docker-password="$(aws ecr get-login-password --region "$region")" \ --dry-run=client -o yaml | "$KUBECTL" apply -f - diff --git a/deployments/k8s/reaperc2/examples/bedrock-irsa.md b/deployments/k8s/reaperc2/examples/bedrock-irsa.md index e7c9701..6d3907e 100644 --- a/deployments/k8s/reaperc2/examples/bedrock-irsa.md +++ b/deployments/k8s/reaperc2/examples/bedrock-irsa.md @@ -6,7 +6,7 @@ Operator AI uses the AWS credential chain when Bedrock is enabled and no static 1. Enable model access in the Bedrock console for the models you need (e.g. Claude Opus 4.7 inference profile in `us-east-1`). -2. Create an IAM policy from `bedrock-iam-policy.json` (replace `123456789012` with your account ID): +2. Create an IAM policy from `bedrock-iam-policy.json` (replace `123456789012` in that file with your account ID): ```bash aws iam create-policy \ @@ -23,7 +23,7 @@ Operator AI uses the AWS credential chain when Bedrock is enabled and no static --namespace=reaperc2-ns \ --name=reaperc2 \ --role-name reaperc2-bedrock \ - --attach-policy-arn arn:aws:iam::123456789012:policy/ReaperC2BedrockInvoke \ + --attach-policy-arn arn:aws:iam:::policy/ReaperC2BedrockInvoke \ --approve \ --override-existing-serviceaccounts ``` @@ -31,7 +31,7 @@ Operator AI uses the AWS credential chain when Bedrock is enabled and no static Or annotate manually after creating the role: ```yaml - eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/reaperc2-bedrock + eks.amazonaws.com/role-arn: arn:aws:iam:::role/reaperc2-bedrock ``` 4. Ensure `reaperc2-ai-config` has `REAPER_AI_BEDROCK_USE_IAM: "1"` (set in `../operator-ai.local.yaml`). diff --git a/deployments/k8s/reaperc2/reroll.sh b/deployments/k8s/reaperc2/reroll.sh index dcd7580..99112fb 100755 --- a/deployments/k8s/reaperc2/reroll.sh +++ b/deployments/k8s/reaperc2/reroll.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Restart ReaperC2 pods and optionally refresh in-cluster config before restart. -# Typical uses: new image already in deployment.yaml / new ECR tag, rotated AI secrets, updated operator-ai.local.yaml. +# Typical uses: new :latest already in ECR (prefer ./ship.sh), rotated AI secrets, updated operator-ai.local.yaml. # # ./reroll.sh # rollout restart + wait (same image/spec already in the cluster) # ./reroll.sh --apply-core # kubectl apply -k overlay (picks up edited base/deployment.yaml, ConfigMaps, etc.), then restart @@ -34,7 +34,7 @@ Environment: Examples: ./reroll.sh - ./reroll.sh --apply-core # new image: line in base/deployment.yaml + this + build/push first + ./reroll.sh --apply-core # apply overlay + restart (no image build; use ./ship.sh to push) kubectl apply -f examples/documentdb-secret.local.yaml && ./reroll.sh ./reroll.sh --apply-secrets ./reroll.sh --refresh-ecr diff --git a/deployments/k8s/reaperc2/ship.sh b/deployments/k8s/reaperc2/ship.sh new file mode 100755 index 0000000..a003ef0 --- /dev/null +++ b/deployments/k8s/reaperc2/ship.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Build, push registry.reaper-ut.com :latest, restart pods so imagePullPolicy: Always pulls the new digest. +# Kubernetes does not watch :latest — a rollout is required after every push. +# +# ./ship.sh # amd64, tag :latest, apply-core + restart +# ./ship.sh --arch both # multi-arch manifest at :latest +# ./ship.sh --push-only # registry only, no kubectl +# ./ship.sh --deploy-only # apply-core + restart, no rebuild +# +# ECR_REGISTRY defaults to registry.reaper-ut.com (same as Makefile / deployment.yaml). + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$HERE" + +ARCH_RAW="${REAPER_IMAGE_ARCH:-amd64}" +PUSH_ONLY=0 +DEPLOY_ONLY=0 +MAKE_ARGS=() + +usage() { + cat <<'EOF' +Usage: ./ship.sh [options] [make-vars...] + + 1. Build ReaperC2 and push to registry.reaper-ut.com as :latest (also aliases the git SHA). + 2. kubectl apply -k the overlay (image: registry.reaper-ut.com/reaperc2:latest) and rollout restart. + +Options: + --arch amd64|arm64|both Image arch (default amd64). Aliases: x86_64, arm, multi. + --push-only Push registry only; do not apply or restart. + --deploy-only Apply overlay + rollout; do not build (image already in the registry). + -h, --help This help. + +Environment: + ECR_REGISTRY, AWS_REGION Default registry.reaper-ut.com / us-east-1. + IMAGE_TAG Default latest. Extra tags: git SHA, and :latest if you override IMAGE_TAG. + AWS_CLI_PROFILE, SCYTHE_GIT_REF, REAPER_NS, REAPER_CLUSTER — same as make / deploy-cluster.sh. + +Examples: + cd deployments/k8s/reaperc2 + ./ship.sh + ./ship.sh --push-only + ./ship.sh --deploy-only +EOF +} + +die() { echo "error: $*" >&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --arch|-a) + [[ $# -ge 2 ]] || die "--arch requires a value" + ARCH_RAW="$2" + shift 2 + ;; + --push-only) PUSH_ONLY=1; shift ;; + --deploy-only) DEPLOY_ONLY=1; shift ;; + -h|--help) usage; exit 0 ;; + *) MAKE_ARGS+=("$1"); shift ;; + esac +done + +export ECR_REGISTRY="${ECR_REGISTRY:-registry.reaper-ut.com}" +export AWS_REGION="${AWS_REGION:-us-east-1}" +export IMAGE_TAG="${IMAGE_TAG:-latest}" + +if [[ "$PUSH_ONLY" -eq 1 && "$DEPLOY_ONLY" -eq 1 ]]; then + die "use only one of --push-only or --deploy-only" +fi + +if [[ "$DEPLOY_ONLY" -eq 1 ]]; then + echo "==> Apply overlay and restart so pods pull :latest" + ./reroll.sh --apply-core + exit 0 +fi + +echo "==> Registry $ECR_REGISTRY tag=$IMAGE_TAG arch=$ARCH_RAW" + +if [[ ${#MAKE_ARGS[@]} -gt 0 ]]; then + ./build-push-image.sh --arch "$ARCH_RAW" "${MAKE_ARGS[@]}" +else + ./build-push-image.sh --arch "$ARCH_RAW" +fi + +if [[ "$PUSH_ONLY" -eq 1 ]]; then + echo "==> Push complete (--push-only). Restart later with: ./reroll.sh --apply-core" + exit 0 +fi + +echo "==> Apply overlay and restart so pods pull :latest" +./reroll.sh --apply-core diff --git a/docs/operator-guide-commands.md b/docs/operator-guide-commands.md index 3999895..d8bb6e0 100644 --- a/docs/operator-guide-commands.md +++ b/docs/operator-guide-commands.md @@ -28,7 +28,7 @@ Shows all engagement beacons and commands waiting for the next heartbeat. **Refr ## Files -Lists **file artifacts**: operator-staged uploads and files **downloaded from beacons** via the `download` built-in. Stored under `REAPER_ARTIFACT_DIR` (default `./data/reaper_artifacts`). Download via `GET /api/beacon-artifacts/{id}/file`. +Lists **file artifacts**: operator-staged uploads and files **downloaded from beacons** via the `download` built-in. Metadata is in MongoDB collection `file_artifacts`; bytes are stored in the same database as GridFS (`reaper_artifacts.files` / `reaper_artifacts.chunks`) so they survive pod recycles and are shared across replicas. Download via `GET /api/beacon-artifacts/{id}/file`. ## Output history diff --git a/pkg/adminpanel/handlers_beacon_files.go b/pkg/adminpanel/handlers_beacon_files.go index c7f2977..b5d624e 100644 --- a/pkg/adminpanel/handlers_beacon_files.go +++ b/pkg/adminpanel/handlers_beacon_files.go @@ -180,7 +180,7 @@ func (s *Server) handleAPIBeaconArtifactFile(w http.ResponseWriter, r *http.Requ jsonError(w, http.StatusForbidden, "artifact not in this engagement") return } - data, err := dbconnections.ReadArtifactBytes(oid) + data, err := dbconnections.ReadArtifactBytes(ctx, oid) if err != nil { log.Printf("admin: read artifact: %v", err) jsonError(w, http.StatusNotFound, "file missing") diff --git a/pkg/dbconnections/file_artifacts.go b/pkg/dbconnections/file_artifacts.go index 13c877c..fc066b3 100644 --- a/pkg/dbconnections/file_artifacts.go +++ b/pkg/dbconnections/file_artifacts.go @@ -1,22 +1,24 @@ package dbconnections import ( + "bytes" "context" + "errors" "fmt" "io" - "os" - "path/filepath" "strings" "time" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/gridfs" "go.mongodb.org/mongo-driver/mongo/options" ) const ( collectionFileArtifacts = "file_artifacts" + gridFSBucketName = "reaper_artifacts" // FileArtifactKindStaging is an operator-uploaded blob waiting to be sent to a beacon. FileArtifactKindStaging = "staging" // FileArtifactKindDownload is a file pulled from a beacon via Scythe download. @@ -26,7 +28,10 @@ const ( // FileArtifactsCollection stores metadata for staged uploads and beacon downloads. var FileArtifactsCollection *mongo.Collection +var fileArtifactsDB *mongo.Database + func initFileArtifactsCollection(db *mongo.Database) { + fileArtifactsDB = db FileArtifactsCollection = db.Collection(collectionFileArtifacts) ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() @@ -35,17 +40,11 @@ func initFileArtifactsCollection(db *mongo.Database) { }) } -// ArtifactStorageRoot is the on-disk root for file bytes (override with REAPER_ARTIFACT_DIR). -func ArtifactStorageRoot() string { - r := os.Getenv("REAPER_ARTIFACT_DIR") - if r != "" { - return r +func artifactBucket() (*gridfs.Bucket, error) { + if fileArtifactsDB == nil { + return nil, fmt.Errorf("file artifacts storage not initialized") } - return filepath.Join(".", "data", "reaper_artifacts") -} - -func artifactPathForID(id primitive.ObjectID) string { - return filepath.Join(ArtifactStorageRoot(), "artifacts", id.Hex()) + return gridfs.NewBucket(fileArtifactsDB, options.GridFSBucket().SetName(gridFSBucketName)) } // FileArtifact is metadata for a staged or downloaded file. @@ -60,33 +59,63 @@ type FileArtifact struct { CreatedAt time.Time `bson:"created_at" json:"created_at"` } -func ensureArtifactDir() error { - root := filepath.Join(ArtifactStorageRoot(), "artifacts") - return os.MkdirAll(root, 0750) +func applyBucketWriteDeadline(bucket *gridfs.Bucket, ctx context.Context) { + if deadline, ok := ctx.Deadline(); ok { + _ = bucket.SetWriteDeadline(deadline) + } } -// WriteStagingArtifact stores an operator upload for later enqueue as a Scythe upload command. -func WriteStagingArtifact(ctx context.Context, clientID, originalName string, r io.Reader, maxBytes int64) (*FileArtifact, error) { - if err := ensureArtifactDir(); err != nil { - return nil, err +func applyBucketReadDeadline(bucket *gridfs.Bucket, ctx context.Context) { + if deadline, ok := ctx.Deadline(); ok { + _ = bucket.SetReadDeadline(deadline) } - id := primitive.NewObjectID() - path := artifactPathForID(id) - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0640) +} + +// writeArtifactBytes stores file bytes in GridFS under the given ObjectID. The GridFS filename is the hex id. +func writeArtifactBytes(ctx context.Context, id primitive.ObjectID, r io.Reader, maxBytes int64) (int64, error) { + bucket, err := artifactBucket() if err != nil { - return nil, err + return 0, err } - n, err := io.Copy(f, io.LimitReader(r, maxBytes+1)) - if closeErr := f.Close(); closeErr != nil && err == nil { - err = closeErr + applyBucketWriteDeadline(bucket, ctx) + us, err := bucket.OpenUploadStreamWithID(id, id.Hex()) + if err != nil { + return 0, err } + n, err := io.Copy(us, io.LimitReader(r, maxBytes+1)) if err != nil { - _ = os.Remove(path) - return nil, err + _ = us.Abort() + return 0, err } if n > maxBytes { - _ = os.Remove(path) - return nil, fmt.Errorf("file larger than %d bytes", maxBytes) + _ = us.Abort() + return 0, fmt.Errorf("file larger than %d bytes", maxBytes) + } + if err := us.Close(); err != nil { + _ = bucket.DeleteContext(ctx, id) + return 0, err + } + return n, nil +} + +func deleteArtifactBytes(ctx context.Context, id primitive.ObjectID) error { + bucket, err := artifactBucket() + if err != nil { + return err + } + err = bucket.DeleteContext(ctx, id) + if err == nil || errors.Is(err, gridfs.ErrFileNotFound) { + return nil + } + return err +} + +// WriteStagingArtifact stores an operator upload for later enqueue as a Scythe upload command. +func WriteStagingArtifact(ctx context.Context, clientID, originalName string, r io.Reader, maxBytes int64) (*FileArtifact, error) { + id := primitive.NewObjectID() + n, err := writeArtifactBytes(ctx, id, r, maxBytes) + if err != nil { + return nil, err } doc := FileArtifact{ ID: id, @@ -102,7 +131,7 @@ func WriteStagingArtifact(ctx context.Context, clientID, originalName string, r ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if _, err := FileArtifactsCollection.InsertOne(ctx, doc); err != nil { - _ = os.Remove(path) + _ = deleteArtifactBytes(ctx, id) return nil, err } return &doc, nil @@ -110,12 +139,9 @@ func WriteStagingArtifact(ctx context.Context, clientID, originalName string, r // WriteDownloadArtifact stores bytes from a Scythe beacon download result. func WriteDownloadArtifact(ctx context.Context, clientID, remotePath string, data []byte) (*FileArtifact, error) { - if err := ensureArtifactDir(); err != nil { - return nil, err - } id := primitive.NewObjectID() - path := artifactPathForID(id) - if err := os.WriteFile(path, data, 0640); err != nil { + n, err := writeArtifactBytes(ctx, id, bytes.NewReader(data), int64(len(data))) + if err != nil { return nil, err } doc := FileArtifact{ @@ -123,7 +149,7 @@ func WriteDownloadArtifact(ctx context.Context, clientID, remotePath string, dat ClientID: clientID, Kind: FileArtifactKindDownload, RemotePath: remotePath, - ByteSize: int64(len(data)), + ByteSize: n, CreatedAt: time.Now().UTC(), } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -132,7 +158,7 @@ func WriteDownloadArtifact(ctx context.Context, clientID, remotePath string, dat doc.EngagementID = strings.TrimSpace(bc.EngagementId) } if _, err := FileArtifactsCollection.InsertOne(ctx, doc); err != nil { - _ = os.Remove(path) + _ = deleteArtifactBytes(ctx, id) return nil, err } return &doc, nil @@ -150,10 +176,18 @@ func FindFileArtifact(ctx context.Context, id primitive.ObjectID) (*FileArtifact return &doc, nil } -// ReadArtifactBytes returns on-disk bytes for an artifact. -func ReadArtifactBytes(id primitive.ObjectID) ([]byte, error) { - path := artifactPathForID(id) - return os.ReadFile(path) +// ReadArtifactBytes returns GridFS bytes for an artifact. +func ReadArtifactBytes(ctx context.Context, id primitive.ObjectID) ([]byte, error) { + bucket, err := artifactBucket() + if err != nil { + return nil, err + } + applyBucketReadDeadline(bucket, ctx) + var buf bytes.Buffer + if _, err := bucket.DownloadToStream(id, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil } // ListFileArtifactsForClient returns newest artifacts for a beacon (staging + download). @@ -191,11 +225,10 @@ func DeleteStagingArtifact(ctx context.Context, id primitive.ObjectID) error { if res.DeletedCount == 0 { return mongo.ErrNoDocuments } - _ = os.Remove(artifactPathForID(id)) - return nil + return deleteArtifactBytes(ctx, id) } -// DeleteArtifactByID removes any artifact row (staging or download) and deletes on-disk bytes if present. +// DeleteArtifactByID removes any artifact row (staging or download) and deletes GridFS bytes if present. func DeleteArtifactByID(ctx context.Context, id primitive.ObjectID) error { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() @@ -206,6 +239,5 @@ func DeleteArtifactByID(ctx context.Context, id primitive.ObjectID) error { if res.DeletedCount == 0 { return mongo.ErrNoDocuments } - _ = os.Remove(artifactPathForID(id)) - return nil + return deleteArtifactBytes(ctx, id) } diff --git a/pkg/dbconnections/file_artifacts_gridfs_test.go b/pkg/dbconnections/file_artifacts_gridfs_test.go new file mode 100644 index 0000000..8bb15a6 --- /dev/null +++ b/pkg/dbconnections/file_artifacts_gridfs_test.go @@ -0,0 +1,193 @@ +package dbconnections + +import ( + "bytes" + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/gridfs" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func setupArtifactMongo(t *testing.T) context.Context { + t.Helper() + uri := os.Getenv("TEST_MONGO_URI") + if uri == "" { + uri = "mongodb://127.0.0.1:27017" + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetServerSelectionTimeout(500*time.Millisecond)) + if err != nil { + t.Skipf("mongo unavailable: %v", err) + } + if err := client.Ping(ctx, nil); err != nil { + _ = client.Disconnect(context.Background()) + t.Skipf("mongo ping failed: %v", err) + } + + prevColl := FileArtifactsCollection + prevDB := fileArtifactsDB + prevClients := ClientCollection + + dbName := "reaperc2_artifact_test_" + primitive.NewObjectID().Hex() + db := client.Database(dbName) + initFileArtifactsCollection(db) + ClientCollection = db.Collection("clients") + + t.Cleanup(func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer dropCancel() + _ = db.Drop(dropCtx) + _ = client.Disconnect(dropCtx) + FileArtifactsCollection = prevColl + fileArtifactsDB = prevDB + ClientCollection = prevClients + }) + return context.Background() +} + +func TestArtifactBucketNotInitialized(t *testing.T) { + prev := fileArtifactsDB + fileArtifactsDB = nil + t.Cleanup(func() { fileArtifactsDB = prev }) + if _, err := artifactBucket(); err == nil { + t.Fatal("expected error when GridFS db is unset") + } +} + +func TestWriteReadDeleteStagingArtifact(t *testing.T) { + ctx := setupArtifactMongo(t) + payload := []byte("staged payload") + doc, err := WriteStagingArtifact(ctx, "client-a", "tool.sh", bytes.NewReader(payload), ScytheMaxFileBytes) + if err != nil { + t.Fatalf("WriteStagingArtifact: %v", err) + } + if doc.Kind != FileArtifactKindStaging || doc.OriginalFilename != "tool.sh" || doc.ByteSize != int64(len(payload)) { + t.Fatalf("metadata: %+v", doc) + } + + got, err := ReadArtifactBytes(ctx, doc.ID) + if err != nil { + t.Fatalf("ReadArtifactBytes: %v", err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("bytes mismatch: %q", got) + } + + listed, err := ListFileArtifactsForClient(ctx, "client-a", 10) + if err != nil { + t.Fatalf("ListFileArtifactsForClient: %v", err) + } + if len(listed) != 1 || listed[0].ID != doc.ID { + t.Fatalf("list: %+v", listed) + } + + if err := DeleteArtifactByID(ctx, doc.ID); err != nil { + t.Fatalf("DeleteArtifactByID: %v", err) + } + if _, err := FindFileArtifact(ctx, doc.ID); !errors.Is(err, mongo.ErrNoDocuments) { + t.Fatalf("metadata after delete: %v", err) + } + if _, err := ReadArtifactBytes(ctx, doc.ID); !errors.Is(err, gridfs.ErrFileNotFound) { + t.Fatalf("gridfs after delete: %v", err) + } +} + +func TestWriteReadDeleteDownloadArtifact(t *testing.T) { + ctx := setupArtifactMongo(t) + payload := []byte("downloaded from beacon") + doc, err := WriteDownloadArtifact(ctx, "client-b", "/tmp/out.bin", payload) + if err != nil { + t.Fatalf("WriteDownloadArtifact: %v", err) + } + if doc.Kind != FileArtifactKindDownload || doc.RemotePath != "/tmp/out.bin" { + t.Fatalf("metadata: %+v", doc) + } + got, err := ReadArtifactBytes(ctx, doc.ID) + if err != nil { + t.Fatalf("ReadArtifactBytes: %v", err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("bytes mismatch: %q", got) + } + if err := DeleteStagingArtifact(ctx, doc.ID); !errors.Is(err, mongo.ErrNoDocuments) { + t.Fatalf("DeleteStagingArtifact on download: %v", err) + } + got, err = ReadArtifactBytes(ctx, doc.ID) + if err != nil { + t.Fatalf("download bytes should remain: %v", err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("bytes after failed staging delete: %q", got) + } + if err := DeleteArtifactByID(ctx, doc.ID); err != nil { + t.Fatalf("DeleteArtifactByID: %v", err) + } +} + +func TestWriteArtifactBytesOversizeLeavesNoGridFSOrphan(t *testing.T) { + ctx := setupArtifactMongo(t) + const chunk = 255 * 1024 + maxBytes := int64(chunk + 100) + payload := bytes.Repeat([]byte("x"), int(maxBytes)+1) + + id := primitive.NewObjectID() + _, err := writeArtifactBytes(ctx, id, bytes.NewReader(payload), maxBytes) + if err == nil || !strings.Contains(err.Error(), "file larger than") { + t.Fatalf("expected oversize error, got %v", err) + } + + bucket, err := artifactBucket() + if err != nil { + t.Fatal(err) + } + files, err := bucket.GetFilesCollection().CountDocuments(ctx, bson.M{"_id": id}) + if err != nil { + t.Fatal(err) + } + if files != 0 { + t.Fatalf("files leftover: %d", files) + } + chunks, err := bucket.GetChunksCollection().CountDocuments(ctx, bson.M{"files_id": id}) + if err != nil { + t.Fatal(err) + } + if chunks != 0 { + t.Fatalf("chunks leftover: %d", chunks) + } + + _, err = WriteStagingArtifact(ctx, "client-c", "big.bin", bytes.NewReader(payload), 8) + if err == nil || !strings.Contains(err.Error(), "file larger than") { + t.Fatalf("WriteStagingArtifact oversize: %v", err) + } + n, err := FileArtifactsCollection.CountDocuments(ctx, bson.M{"client_id": "client-c"}) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatalf("metadata leftover: %d", n) + } +} + +func TestWriteStagingArtifactEmptyFile(t *testing.T) { + ctx := setupArtifactMongo(t) + doc, err := WriteStagingArtifact(ctx, "client-d", "empty.bin", bytes.NewReader(nil), ScytheMaxFileBytes) + if err != nil { + t.Fatalf("empty file: %v", err) + } + got, err := ReadArtifactBytes(ctx, doc.ID) + if err != nil { + t.Fatalf("ReadArtifactBytes: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected empty bytes, got %d", len(got)) + } +} diff --git a/pkg/dbconnections/staging_upload_expand.go b/pkg/dbconnections/staging_upload_expand.go index ebe56ab..ade21a2 100644 --- a/pkg/dbconnections/staging_upload_expand.go +++ b/pkg/dbconnections/staging_upload_expand.go @@ -42,7 +42,7 @@ func expandStagingUploadCommands(ctx context.Context, clientID string, cmds []in if err != nil || meta.Kind != FileArtifactKindStaging || meta.ClientID != clientID { return nil, fmt.Errorf("upload: staging artifact not found for this beacon") } - raw, err := ReadArtifactBytes(oid) + raw, err := ReadArtifactBytes(ctx, oid) if err != nil { return nil, fmt.Errorf("upload: read staging file: %w", err) } diff --git a/scripts/aws-for-make.sh b/scripts/aws-for-make.sh index 3b44fe9..40324e7 100755 --- a/scripts/aws-for-make.sh +++ b/scripts/aws-for-make.sh @@ -7,7 +7,7 @@ # 3. AWS_PROFILE — named profile from the shell # 4. Default credential chain (instance role, SSO cache, etc.) # -# Account id belongs in Makefile AWS_ACCOUNT_ID, not in AWS_PROFILE. +# Do not put an AWS account id in AWS_PROFILE. set -euo pipefail From c0ae98a337e1f378f1f1bbfd1183a81fb59e0a13 Mon Sep 17 00:00:00 2001 From: BuildAndDestroy Date: Wed, 9 Sep 2026 10:51:32 -0600 Subject: [PATCH 2/2] Bump golang.org/x/crypto to v0.55.0 for CVE-2026-56854. Trivy flagged the CRITICAL advisory on v0.54.0; the fix is in 0.55.0. Co-authored-by: Cursor --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 46ec52d..9d671c8 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pquerna/otp v1.5.0 github.com/yuin/goldmark v1.7.17 go.mongodb.org/mongo-driver v1.17.7 - golang.org/x/crypto v0.54.0 + golang.org/x/crypto v0.55.0 ) require ( @@ -41,5 +41,5 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 39007a0..eab1131 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ go.mongodb.org/mongo-driver v1.17.7 h1:a9w+U3Vt67eYzcfq3k/OAv284/uUUkL0uP75VE5rC go.mongodb.org/mongo-driver v1.17.7/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -94,8 +94,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=