ops: promote deploy stuff

.github/helpers/deploy.sh

@@ -2,6 +2,13 @@
 
 # this script deploys a desk to a ship from a github repository
 # assumes gcloud credentials are loaded and gcloud installed.
+#
+# The desk is assembled HERE (in the runner): we clone the repo at $ref, vendor
+# its desk dependencies with peru (see peru.yaml — a curated subset of base-dev)
+# and ship the finished desk down to the ship VM. The remote only unpacks it and
+# commits — it needs no peru, no git, and no urbit clone. This replaces rsyncing
+# ALL of pkg/base-dev onto the desk (the new kernel builds every mark present,
+# and many unused base-dev marks don't compile on the target kelvin).
 
 repo=$1
 desk=$2
@@ -11,31 +18,77 @@ project=$5
 ref=${6:-develop}
 folder=$ship/$desk
 
+echo "Deploying $desk from $ref of $repo to $ship in $zone of $project"
 set -e
 set -o pipefail
-cmdfile=$(mktemp "${TMPDIR:-/tmp/}janeway.XXXXXXXXX")
-# mktemp only used for generating a random folder name below
-cmds='
-source_repo=$(mktemp --dry-run /tmp/repo.janeway.XXXXXXXXX)
-git clone --depth 1 --branch '$ref' https://github.com/'$repo'.git $source_repo
-urbit_repo=$(mktemp --dry-run /tmp/repo.urbit.XXXXXXXXX)
-git clone --depth 1 https://github.com/urbit/urbit.git $urbit_repo -b '$URBIT_REPO_TAG' --single-branch
-cd $source_repo
-cd /urbit || return
-curl -s --data '\''{"source":{"dojo":"+hood/mount %'$desk'"},"sink":{"app":"hood"}}'\'' http://localhost:12321
-rsync -avL --delete $urbit_repo/pkg/base-dev/ '$folder'
-rsync -avL $source_repo/desk/ '$folder'
-curl -s --data '\''{"source":{"dojo":"+hood/commit %'$desk'"},"sink":{"app":"hood"}}'\'' http://localhost:12321
-rm -rf $source_repo
-rm -rf $urbit_repo
-'
-echo "$cmds" >> "$cmdfile"
+
+# --- Assemble the desk in the runner ---------------------------------------
+workdir=$(mktemp -d "${TMPDIR:-/tmp/}janeway.XXXXXXXXX")
+trap 'rm -rf "$workdir"' EXIT
+
+git clone --depth 1 --branch "$ref" "https://github.com/$repo.git" "$workdir/src"
+
+# Install peru if it isn't already available (assemble-desk.sh runs peru sync).
+if ! command -v peru >/dev/null 2>&1; then
+  echo "Installing peru..."
+  pipx install peru \
+    || pip install --user peru \
+    || pip3 install --user --break-system-packages peru
+  export PATH="$HOME/.local/bin:$PATH"
+fi
+
+# Assemble desk-deps/ (peru-vendored) + desk/ (our source) into a staging dir.
+"$workdir/src/scripts/assemble-desk.sh" "$workdir/assembled"
+
+# Package the assembled, self-contained desk.
+tar czf "$workdir/desk.tgz" -C "$workdir" assembled
+
+# --- Speed up the IAP tunnel ------------------------------------------------
+# gcloud's IAP TCP forwarding is a single-threaded Python websocket proxy. With
+# NumPy available its upload path is dramatically faster (gcloud prints a
+# warning recommending this). The whole assembled desk is shipped UP through
+# this tunnel, so without NumPy the scp below takes ~4 min. CLOUDSDK_PYTHON_-
+# SITEPACKAGES lets gcloud's bundled interpreter see the system-installed numpy.
+export CLOUDSDK_PYTHON_SITEPACKAGES=1
+if ! python3 -c 'import numpy' >/dev/null 2>&1; then
+  echo "Installing NumPy to speed up the IAP tunnel..."
+  pip3 install --user numpy \
+    || pip3 install --user --break-system-packages numpy \
+    || echo "WARNING: NumPy install failed; tunnel upload will be slow."
+fi
+
+# --- SSH key setup ----------------------------------------------------------
 sshpriv=$(mktemp "${TMPDIR:-/tmp/}ssh.XXXXXXXXX")
 sshpub=$sshpriv.pub
 echo "$SSH_PUB_KEY" >> "$sshpub"
 echo "$SSH_SEC_KEY" >> "$sshpriv"
-chmod 600 $sshpub
-chmod 600 $sshpriv
+chmod 600 "$sshpub"
+chmod 600 "$sshpriv"
+
+# --- Ship the assembled desk to the remote over the IAP tunnel --------------
+gcloud compute scp \
+  --project "$project" \
+  --tunnel-through-iap \
+  --ssh-key-file "$sshpriv" \
+  --zone "$zone" --verbosity info \
+  "$workdir/desk.tgz" urb@"$ship":/tmp/janeway-desk.tgz
+
+# --- Remote: mount, unpack into clay, commit --------------------------------
+# $desk and $folder are expanded here (in the runner); \$staging is deferred to
+# the remote. No clones / peru / base-dev rsync on the ship.
+cmdfile=$(mktemp "${TMPDIR:-/tmp/}janeway.XXXXXXXXX")
+cat > "$cmdfile" <<EOF
+staging=\$(mktemp -d)
+tar xzf /tmp/janeway-desk.tgz -C \$staging
+cd /urbit || exit 1
+curl -s --data '{"source":{"dojo":"+hood/unmount %$desk"},"sink":{"app":"hood"}}' http://localhost:12321
+curl -s --data '{"source":{"dojo":"+hood/mount %$desk"},"sink":{"app":"hood"}}' http://localhost:12321
+rsync -avL --delete \$staging/assembled/ $folder
+curl -s --data '{"source":{"dojo":"+hood/commit %$desk"},"sink":{"app":"hood"}}' http://localhost:12321
+rm -rf \$staging /tmp/janeway-desk.tgz
+EOF
+echo "Remote commands:"
+cat "$cmdfile"
 
 gcloud compute \
   --project "$project" \
@@ -46,4 +99,4 @@ gcloud compute \
   --zone "$zone" --verbosity info \
   urb@"$ship" < "$cmdfile"
 
-echo "OTA performed for $desk on $ship"
+echo "OTA performed for $desk on $ship"

.github/workflows/deploy-canary.yml

@@ -28,4 +28,3 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}

.github/workflows/deploy-external.yml

@@ -30,12 +30,10 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}
       - id: deploy-kernel
         name: Deploy Kernel Moon
         if: ${{ github.event.inputs.kernel == 'true' }}
         run: ./.github/helpers/deploy.sh tloncorp/landscape landscape bosdev-dozzod-marnus us-central1-b mainnet-tlon-other-2d ${{ github.event.inputs.tag }}
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}

.github/workflows/deploy-internal.yml

@@ -25,4 +25,3 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}

.github/workflows/deploy-kernel.yml

@@ -80,4 +80,3 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG_KERNEL }}

.github/workflows/deploy-live.yml

@@ -26,4 +26,3 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}

.github/workflows/deploy.yml

@@ -83,4 +83,3 @@ jobs:
         env:
           SSH_SEC_KEY: ${{ secrets.GCP_SSH_SEC_KEY }}
           SSH_PUB_KEY: ${{ secrets.GCP_SSH_PUB_KEY }}
-          URBIT_REPO_TAG: ${{ vars.URBIT_REPO_TAG }}

.gitignore

@@ -8,4 +8,13 @@ stats.html
 .vercel
 .vscode
 
-gha-creds-*
+gha-creds-*
+
+# Vendored desk dependencies — fetched into desk-deps/ by scripts/sync-deps.sh
+# (see peru.yaml). Kept out of git so upstream stays pinned but out of history;
+# assembled into a desk by scripts/assemble-desk.sh (deps first, then desk/).
+/desk-deps/
+
+# peru state
+/.peru/
+/peru.lock

README.md

@@ -14,6 +14,29 @@ To develop against a working ship, you first need to add a `.env.local` file to
 
 Regardless of what you run to develop, Vite will hot-reload code changes as you work so you don't have to constantly refresh.
 
+## Desk dependencies (peru)
+
+The Hoon `desk/` holds only landscape's own source. Its upstream dependencies (a
+curated subset of urbit's `pkg/base-dev` — `default-agent`, `dbug`, `server`,
+the standard marks, etc.) are vendored by [peru] into a separate, **gitignored**
+`desk-deps/` tree, per `peru.yaml`. This replaced rsyncing _all_ of `pkg/base-dev`,
+which pulled in unused files that the new kernel — which builds every mark on a
+desk — fails to compile.
+
+- A fresh checkout's `desk/{lib,sur,mar}` is intentionally **missing** the
+  standard files (`default-agent`, `dbug`, `server`, `hoon`, `json`, …) — they
+  live in `desk-deps/` after a sync. Don't "fix" this by committing them.
+- `./scripts/sync-deps.sh` (= `peru sync`) — populate `desk-deps/`. Run after
+  cloning and after editing `peru.yaml`. Requires peru (`pipx install peru`).
+- `./scripts/assemble-desk.sh <target>` — assemble a full desk into `<target>`:
+  rsync `desk-deps/` in with `--delete`, then `desk/` on top, then stamp
+  `commit.txt`. This is how the `%landscape` desk is built for a ship (used by
+  `.github/helpers/deploy.sh` and for local dev).
+- The upstream kernel rev is **pinned in `peru.yaml`**. To build against a
+  different kernel, change `rev` there on your branch.
+
+[peru]: https://github.com/buildinspace/peru
+
 ## Deploying
 
 Deploys to internal moons are managed via github actions.