From 4fdaf2c9367e5c916680288b4056ad5ba310e8f6 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Thu, 18 Jun 2026 13:34:03 -0500 Subject: [PATCH 1/3] Vendor a curated base-dev subset via peru instead of dumping all of it The new kernel builds every mark present on a desk, so rsyncing all of urbit's pkg/base-dev (~92 files) onto the landscape desk pulled in unused marks that fail to compile. Vendor only the transitive closure of desk/'s imports via peru (peru.yaml, pinned to 408k-rc2) into a gitignored desk-deps/, and assemble desk-deps/ + desk/ in the deploy runner. - peru.yaml: 10 lib, 2 sur, 10 mar picks from base-dev - scripts/sync-deps.sh, scripts/assemble-desk.sh - deploy.sh: assemble in runner, scp tarball, remote only mounts/commits - drop now-unused URBIT_REPO_TAG from deploy workflows (rev pinned in peru.yaml) - delete dead files desk/lib/graph-store.hoon, desk/sur/activity.hoon Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/helpers/deploy.sh | 78 ++- .github/workflows/deploy-canary.yml | 1 - .github/workflows/deploy-external.yml | 2 - .github/workflows/deploy-internal.yml | 1 - .github/workflows/deploy-kernel.yml | 1 - .github/workflows/deploy-live.yml | 1 - .github/workflows/deploy.yml | 1 - .gitignore | 11 +- README.md | 23 + desk/lib/graph-store.hoon | 919 -------------------------- desk/sur/activity.hoon | 368 ----------- peru.yaml | 60 ++ scripts/assemble-desk.sh | 44 ++ scripts/sync-deps.sh | 22 + 14 files changed, 217 insertions(+), 1315 deletions(-) delete mode 100644 desk/lib/graph-store.hoon delete mode 100644 desk/sur/activity.hoon create mode 100644 peru.yaml create mode 100755 scripts/assemble-desk.sh create mode 100755 scripts/sync-deps.sh diff --git a/.github/helpers/deploy.sh b/.github/helpers/deploy.sh index 439a9d24..ebf5b803 100755 --- a/.github/helpers/deploy.sh +++ b/.github/helpers/deploy.sh @@ -2,6 +2,13 @@ # this script deploys a desk to a ship from a github repository # assumes gcloud credentials are loaded and gcloud installed. +# +# The desk is assembled HERE (in the runner): we clone the repo at $ref, vendor +# its desk dependencies with peru (see peru.yaml — a curated subset of base-dev) +# and ship the finished desk down to the ship VM. The remote only unpacks it and +# commits — it needs no peru, no git, and no urbit clone. This replaces rsyncing +# ALL of pkg/base-dev onto the desk (the new kernel builds every mark present, +# and many unused base-dev marks don't compile on the target kelvin). repo=$1 desk=$2 @@ -11,31 +18,62 @@ project=$5 ref=${6:-develop} folder=$ship/$desk +echo "Deploying $desk from $ref of $repo to $ship in $zone of $project" set -e set -o pipefail -cmdfile=$(mktemp "${TMPDIR:-/tmp/}janeway.XXXXXXXXX") -# mktemp only used for generating a random folder name below -cmds=' -source_repo=$(mktemp --dry-run /tmp/repo.janeway.XXXXXXXXX) -git clone --depth 1 --branch '$ref' https://github.com/'$repo'.git $source_repo -urbit_repo=$(mktemp --dry-run /tmp/repo.urbit.XXXXXXXXX) -git clone --depth 1 https://github.com/urbit/urbit.git $urbit_repo -b '$URBIT_REPO_TAG' --single-branch -cd $source_repo -cd /urbit || return -curl -s --data '\''{"source":{"dojo":"+hood/mount %'$desk'"},"sink":{"app":"hood"}}'\'' http://localhost:12321 -rsync -avL --delete $urbit_repo/pkg/base-dev/ '$folder' -rsync -avL $source_repo/desk/ '$folder' -curl -s --data '\''{"source":{"dojo":"+hood/commit %'$desk'"},"sink":{"app":"hood"}}'\'' http://localhost:12321 -rm -rf $source_repo -rm -rf $urbit_repo -' -echo "$cmds" >> "$cmdfile" + +# --- Assemble the desk in the runner --------------------------------------- +workdir=$(mktemp -d "${TMPDIR:-/tmp/}janeway.XXXXXXXXX") +trap 'rm -rf "$workdir"' EXIT + +git clone --depth 1 --branch "$ref" "https://github.com/$repo.git" "$workdir/src" + +# Install peru if it isn't already available (assemble-desk.sh runs peru sync). +if ! command -v peru >/dev/null 2>&1; then + echo "Installing peru..." + pipx install peru \ + || pip install --user peru \ + || pip3 install --user --break-system-packages peru + export PATH="$HOME/.local/bin:$PATH" +fi + +# Assemble desk-deps/ (peru-vendored) + desk/ (our source) into a staging dir. +"$workdir/src/scripts/assemble-desk.sh" "$workdir/assembled" + +# Package the assembled, self-contained desk. +tar czf "$workdir/desk.tgz" -C "$workdir" assembled + +# --- SSH key setup ---------------------------------------------------------- sshpriv=$(mktemp "${TMPDIR:-/tmp/}ssh.XXXXXXXXX") sshpub=$sshpriv.pub echo "$SSH_PUB_KEY" >> "$sshpub" echo "$SSH_SEC_KEY" >> "$sshpriv" -chmod 600 $sshpub -chmod 600 $sshpriv +chmod 600 "$sshpub" +chmod 600 "$sshpriv" + +# --- Ship the assembled desk to the remote over the IAP tunnel -------------- +gcloud compute scp \ + --project "$project" \ + --tunnel-through-iap \ + --ssh-key-file "$sshpriv" \ + --zone "$zone" --verbosity info \ + "$workdir/desk.tgz" urb@"$ship":/tmp/janeway-desk.tgz + +# --- Remote: mount, unpack into clay, commit -------------------------------- +# $desk and $folder are expanded here (in the runner); \$staging is deferred to +# the remote. No clones / peru / base-dev rsync on the ship. +cmdfile=$(mktemp "${TMPDIR:-/tmp/}janeway.XXXXXXXXX") +cat > "$cmdfile" <` — assemble a full desk into ``: + rsync `desk-deps/` in with `--delete`, then `desk/` on top, then stamp + `commit.txt`. This is how the `%landscape` desk is built for a ship (used by + `.github/helpers/deploy.sh` and for local dev). +- The upstream kernel rev is **pinned in `peru.yaml`**. To build against a + different kernel, change `rev` there on your branch. + +[peru]: https://github.com/buildinspace/peru + ## Deploying Deploys to internal moons are managed via github actions. diff --git a/desk/lib/graph-store.hoon b/desk/lib/graph-store.hoon deleted file mode 100644 index df11d034..00000000 --- a/desk/lib/graph-store.hoon +++ /dev/null @@ -1,919 +0,0 @@ -/- sur=graph-store, pos=post, pull-hook, hark=hark-store -/+ res=resource, migrate -=< [sur .] -=< [pos .] -=, sur -=, pos -|% -++ hark-content - |= =content - ^- content:hark - ?- -.content - %text content - %mention ship+ship.content - %url text+url.content - %code text+'A code excerpt' - %reference text+'A reference' - == -:: -++ hark-contents - |= cs=(list content) - (turn cs hark-content) -:: NOTE: move these functions to zuse -++ nu :: parse number as hex - |= jon=json - ?> ?=([%s *] jon) - (rash p.jon hex) -:: -++ re :: recursive reparsers - |* [gar=* sef=_|.(fist:dejs-soft:format)] - |= jon=json - ^- (unit _gar) - =- ~! gar ~! (need -) - - ((sef) jon) -:: -++ dank :: tank - ^- $-(json (unit tank)) - =, ^? dejs-soft:format - %+ re *tank |. ~+ - %- of :~ - leaf+sa - palm+(ot style+(ot mid+sa cap+sa open+sa close+sa ~) lines+(ar dank) ~) - rose+(ot style+(ot mid+sa open+sa close+sa ~) lines+(ar dank) ~) - == -:: -++ orm ((on atom node) gth) -++ orm-log ((on time logged-update) gth) -:: -++ enjs - =, enjs:format - |% - :: - ++ signatures - |= s=^signatures - ^- json - [%a (turn ~(tap in s) signature)] - :: - ++ signature - |= s=^signature - ^- json - %- pairs - :~ [%signature s+(scot %ux p.s)] - [%ship (ship q.s)] - [%life (numb r.s)] - == - :: - ++ index - |= ind=^index - ^- json - :- %s - ?: =(~ ind) - '/' - %+ roll ind - |= [cur=@ acc=@t] - ^- @t - =/ num (numb cur) - ?> ?=(%n -.num) - (rap 3 acc '/' p.num ~) - :: - ++ uid - |= u=^uid - ^- json - %- pairs - :~ [%resource (enjs:res resource.u)] - [%index (index index.u)] - == - :: - ++ content - |= c=^content - ^- json - ?- -.c - %mention (frond %mention (ship ship.c)) - %text (frond %text s+text.c) - %url (frond %url s+url.c) - %reference (frond %reference (reference +.c)) - %code - %+ frond %code - %- pairs - :- [%expression s+expression.c] - :_ ~ - :- %output - :: virtualize output rendering, +tank:enjs:format might crash - :: - =/ result=(each (list json) tang) - (mule |.((turn output.c tank))) - ?- -.result - %& a+p.result - %| a+[a+[%s '[[output rendering error]]']~]~ - == - == - :: - ++ reference - |= ref=^reference - |^ - %+ frond -.ref - ?- -.ref - %graph (graph +.ref) - %group (group +.ref) - %app (app +.ref) - == - :: - ++ graph - |= [grp=res gra=res idx=^index] - %- pairs - :~ graph+s+(enjs-path:res gra) - group+s+(enjs-path:res grp) - index+(index idx) - == - :: - ++ group - |= grp=res - s+(enjs-path:res grp) - :: - ++ app - |= [=^ship =desk p=^path] - %- pairs - :~ ship+s+(scot %p ship) - desk+s+desk - path+(path p) - == - -- - :: - ++ maybe-post - |= mp=^maybe-post - ^- json - ?- -.mp - %| s+(scot %ux p.mp) - %& (post p.mp) - == - :: - ++ post - |= p=^post - ^- json - %- pairs - :~ [%author (ship author.p)] - [%index (index index.p)] - [%time-sent (time time-sent.p)] - [%contents [%a (turn contents.p content)]] - [%hash ?~(hash.p ~ s+(scot %ux u.hash.p))] - [%signatures (signatures signatures.p)] - == - :: - ++ update - |= upd=^update - ^- json - |^ (frond %graph-update (pairs ~[(encode q.upd)])) - :: - ++ encode - |= upd=action - ^- [cord json] - ?- -.upd - %add-graph - :- %add-graph - %- pairs - :~ [%resource (enjs:res resource.upd)] - [%graph (graph graph.upd)] - [%mark ?~(mark.upd ~ s+u.mark.upd)] - [%overwrite b+overwrite.upd] - == - :: - %remove-graph - [%remove-graph (enjs:res resource.upd)] - :: - %add-nodes - :- %add-nodes - %- pairs - :~ [%resource (enjs:res resource.upd)] - [%nodes (nodes nodes.upd)] - == - :: - %remove-posts - :- %remove-posts - %- pairs - :~ [%resource (enjs:res resource.upd)] - [%indices (indices indices.upd)] - == - :: - %add-signatures - :- %add-signatures - %- pairs - :~ [%uid (uid uid.upd)] - [%signatures (signatures signatures.upd)] - == - :: - %remove-signatures - :- %remove-signatures - %- pairs - :~ [%uid (uid uid.upd)] - [%signatures (signatures signatures.upd)] - == - :: - %add-tag - :- %add-tag - %- pairs - :~ [%term s+term.upd] - [%uid (uid uid.upd)] - == - :: - %remove-tag - :- %remove-tag - %- pairs - :~ [%term s+term.upd] - [%uid (uid uid.upd)] - == - :: - %archive-graph - [%archive-graph (enjs:res resource.upd)] - :: - %unarchive-graph - [%unarchive-graph (enjs:res resource.upd)] - :: - %keys - [%keys [%a (turn ~(tap in resources.upd) enjs:res)]] - :: - %tags - [%tags [%a (turn ~(tap in tags.upd) |=(=term s+term))]] - :: - %run-updates - [%run-updates ~] - :: - %tag-queries - :- %tag-queries - %- pairs - %+ turn ~(tap by tag-queries.upd) - |= [=term uids=(set ^uid)] - ^- [cord json] - [term [%a (turn ~(tap in uids) uid)]] - == - :: - ++ graph - |= g=^graph - ^- json - %- pairs - %+ turn - (tap:orm g) - |= [a=atom n=^node] - ^- [@t json] - :_ (node n) - =/ idx (numb a) - ?> ?=(%n -.idx) - p.idx - :: - ++ node - |= n=^node - ^- json - %- pairs - :~ [%post (maybe-post post.n)] - :- %children - ?- -.children.n - %empty ~ - %graph (graph +.children.n) - == - == - :: - ++ nodes - |= m=(map ^index ^node) - ^- json - %- pairs - %+ turn ~(tap by m) - |= [n=^index o=^node] - ^- [@t json] - :_ (node o) - =/ idx (index n) - ?> ?=(%s -.idx) - p.idx - :: - ++ indices - |= i=(set ^index) - ^- json - [%a (turn ~(tap in i) index)] - :: - -- - -- -:: -++ dejs - =, dejs:format - |% - ++ update - |= jon=json - ^- ^update - :- *time - ^- action - =< (decode jon) - |% - ++ decode - %- of - :~ [%add-nodes add-nodes] - [%remove-posts remove-posts] - [%add-signatures add-signatures] - [%remove-signatures remove-signatures] - :: - [%add-graph add-graph] - [%remove-graph remove-graph] - :: - [%add-tag add-tag] - [%remove-tag remove-tag] - :: - [%archive-graph archive-graph] - [%unarchive-graph unarchive-graph] - [%run-updates run-updates] - :: - [%keys keys] - [%tags tags] - [%tag-queries tag-queries] - == - :: - ++ add-graph - %- ot - :~ [%resource dejs:res] - [%graph graph] - [%mark (mu so)] - [%overwrite bo] - == - :: - ++ graph - |= a=json - ^- ^graph - =/ or-mp ((on atom ^node) gth) - %+ gas:or-mp ~ - %+ turn ~(tap by ((om node) a)) - |* [b=cord c=*] - ^- [atom ^node] - => .(+< [b c]=+<) - [(rash b dem) c] - :: - ++ remove-graph (ot [%resource dejs:res]~) - ++ archive-graph (ot [%resource dejs:res]~) - ++ unarchive-graph (ot [%resource dejs:res]~) - :: - ++ add-nodes - %- ot - :~ [%resource dejs:res] - [%nodes nodes] - == - :: - ++ nodes (op ;~(pfix fas (more fas dem)) node) - :: - ++ node - %- ot - :~ [%post maybe-post] - [%children internal-graph] - == - :: - ++ internal-graph - |= jon=json - ^- ^internal-graph - ?~ jon - [%empty ~] - [%graph (graph jon)] - :: - ++ maybe-post - |= jon=json - ^- ^maybe-post - ?~ jon !! - ?+ -.jon !! - %s [%| (nu jon)] - %o [%& (post jon)] - == - :: - ++ post - %- ot - :~ [%author (su ;~(pfix sig fed:ag))] - [%index index] - [%time-sent di] - [%contents (ar content)] - [%hash (mu nu)] - [%signatures (as signature)] - == - :: - ++ content - %- of - :~ [%mention (su ;~(pfix sig fed:ag))] - [%text so] - [%url so] - [%reference reference] - [%code eval] - == - :: - ++ reference - |^ - %- of - :~ graph+graph - group+dejs-path:res - app+app - == - :: - ++ graph - %- ot - :~ group+dejs-path:res - graph+dejs-path:res - index+index - == - :: - ++ app - %- ot - :~ ship+(su ;~(pfix sig fed:ag)) - desk+so - path+pa - == - -- - :: - ++ tang - |= jon=^json - ^- ^tang - ?> ?=(%a -.jon) - %- zing - %+ turn - p.jon - |= jo=^json - ^- (list tank) - ?> ?=(%a -.jo) - %+ turn - p.jo - |= j=^json - ?> ?=(%s -.j) - ^- tank - leaf+(trip p.j) - :: - ++ eval - %- ot - :~ expression+so - output+tang - == - :: - ++ remove-posts - %- ot - :~ [%resource dejs:res] - [%indices (as index)] - == - :: - ++ add-signatures - %- ot - :~ [%uid uid] - [%signatures (as signature)] - == - :: - ++ remove-signatures - %- ot - :~ [%uid uid] - [%signatures (as signature)] - == - :: - ++ signature - %- ot - :~ [%hash nu] - [%ship (su ;~(pfix sig fed:ag))] - [%life ni] - == - :: - ++ uid - %- ot - :~ [%resource dejs:res] - [%index index] - == - :: - ++ index (su ;~(pfix fas (more fas dem))) - :: - ++ add-tag - %- ot - :~ [%term so] - [%uid uid] - == - :: - ++ remove-tag - %- ot - :~ [%term so] - [%uid uid] - == - :: - ++ keys - |= =json - *resources - :: - ++ tags - |= =json - *(set term) - :: - ++ tag-queries - |= =json - *^tag-queries - :: - ++ run-updates - |= a=json - ^- [resource update-log] - [*resource *update-log] - -- - ++ pa - |= j=json - ^- path - ?> ?=(%s -.j) - ?: =('/' p.j) / - (stab p.j) - :: - -- -:: -++ create - |_ [our=ship now=time] - ++ post - |= [=index contents=(list content)] - ^- ^post - :* our - index - now - contents - ~ - *signatures - == - -- -:: -++ upgrade - |% - :: - :: +two - :: - ++ marked-graph-to-two - |= [=graph:one m=(unit mark)] - [(graph-to-two graph) m] - :: - ++ graph-to-two - |= =graph:one - (graph:(upgrade ,post:one ,maybe-post:two) graph post-to-two) - :: - ++ post-to-two - |= p=post:one - ^- maybe-post:two - [%& p] - :: - :: - :: +one - :: - ++ update-log-to-one - |= =update-log:zero - ^- update-log:one - %+ gas:orm-log:one *update-log:one - %+ turn (tap:orm-log:zero update-log) - |= [=time =logged-update:zero] - ^- [^time logged-update:one] - :- time - :- p.logged-update - (logged-update-to-one q.logged-update) - :: - ++ logged-update-to-one - |= upd=logged-update-0:zero - ^- logged-action:one - ?+ -.upd upd - %add-graph upd(graph (graph-to-one graph.upd)) - %add-nodes upd(nodes (~(run by nodes.upd) node-to-one)) - == - :: - ++ node-to-one - |= =node:zero - (node:(upgrade ,post:zero ,post:one) node post-to-one) - :: - ++ graph-to-one - |= =graph:zero - (graph:(upgrade ,post:zero ,post:one) graph post-to-one) - :: - ++ marked-graph-to-one - |= [=graph:zero m=(unit mark)] - [(graph-to-one graph) m] - :: - ++ post-to-one - |= p=post:zero - ^- post:one - p(contents (contents-to-one contents.p)) - :: - ++ contents-to-one - |= cs=(list content:zero) - ^- (list content:one) - %+ murn cs - |= =content:zero - ^- (unit content:one) - ?: ?=(%reference -.content) ~ - `content - :: - ++ upgrade - |* [in-pst=mold out-pst=mold] - => - |% - ++ in-orm - ((on atom in-node) gth) - +$ in-node - [post=in-pst children=in-internal-graph] - +$ in-graph - ((mop atom in-node) gth) - +$ in-internal-graph - $~ [%empty ~] - $% [%graph p=in-graph] - [%empty ~] - == - :: - ++ out-orm - ((on atom out-node) gth) - +$ out-node - [post=out-pst children=out-internal-graph] - +$ out-graph - ((mop atom out-node) gth) - +$ out-internal-graph - $~ [%empty ~] - $% [%graph p=out-graph] - [%empty ~] - == - -- - |% - :: - ++ graph - |= $: gra=in-graph - fn=$-(in-pst out-pst) - == - ^- out-graph - %+ gas:out-orm *out-graph - ^- (list [atom out-node]) - %+ turn (tap:in-orm gra) - |= [a=atom n=in-node] - ^- [atom out-node] - [a (node n fn)] - :: - ++ node - |= [nod=in-node fn=$-(in-pst out-pst)] - ^- out-node - :- (fn post.nod) - ^- out-internal-graph - ?: ?=(%empty -.children.nod) - [%empty ~] - [%graph (graph p.children.nod fn)] - -- - :: - ++ zero-load - :: =* infinitely recurses - =, store=zero - =, orm=orm:zero - =, orm-log=orm-log:zero - |% - ++ change-revision-graph - |= [=graph:store q=(unit mark)] - ^- [graph:store (unit mark)] - |^ - :_ q - ?+ q graph - [~ %graph-validator-link] convert-links - [~ %graph-validator-publish] convert-publish - == - :: - ++ convert-links - %+ gas:orm *graph:store - %+ turn (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :: top-level - :: - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^atom =node:store] - ^- [^^atom node:store] - :: existing comments get turned into containers for revisions - :: - :^ atom - post.node(contents ~, hash ~) - %graph - %+ gas:orm *graph:store - :_ ~ :- %0 - :_ [%empty ~] - post.node(index (snoc index.post.node atom), hash ~) - :: - ++ convert-publish - %+ gas:orm *graph:store - %+ turn (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :: top-level - :: - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^atom =node:store] - ^- [^^atom node:store] - :: existing container for publish note revisions - :: - ?+ atom !! - %1 [atom node] - %2 - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^^atom =node:store] - ^- [^^^atom node:store] - :+ atom post.node(contents ~, hash ~) - :- %graph - %+ gas:orm *graph:store - :_ ~ :- %1 - :_ [%empty ~] - post.node(index (snoc index.post.node atom), hash ~) - == - -- - :: - ++ maybe-unix-to-da - |= =atom - ^- @ - :: (bex 127) is roughly 226AD - ?. (lte atom (bex 127)) - atom - (add ~1970.1.1 (div (mul ~s1 atom) 1.000)) - :: - ++ convert-unix-timestamped-node - |= =node:store - ^- node:store - =. index.post.node - (convert-unix-timestamped-index index.post.node) - ?. ?=(%graph -.children.node) - node - :+ post.node - %graph - (convert-unix-timestamped-graph p.children.node) - :: - ++ convert-unix-timestamped-index - |= =index:store - (turn index maybe-unix-to-da) - :: - ++ convert-unix-timestamped-graph - |= =graph:store - %+ gas:orm *graph:store - %+ turn - (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :- (maybe-unix-to-da atom) - (convert-unix-timestamped-node node) - -- - -- -++ import - |= [arc=* our=ship] - ^- (quip card:agent:gall [%6 network]) - |^ - =/ sty [%6 (remake-network ;;(tree-network +.arc))] - :_ sty - %+ turn ~(tap by graphs.sty) - |= [rid=resource =marked-graph] - ^- card:agent:gall - ?: =(our entity.rid) - =/ =cage [%push-hook-action !>([%add rid])] - [%pass / %agent [our %graph-push-hook] %poke cage] - (try-rejoin rid 0) - :: - +$ tree-network - $: graphs=tree-graphs - tag-queries=(tree [term (tree uid)]) - update-logs=tree-update-logs - archive=tree-graphs - ~ - == - +$ tree-graphs (tree [resource tree-marked-graph]) - +$ tree-marked-graph [p=tree-graph q=(unit ^mark)] - +$ tree-graph (tree [atom tree-node]) - +$ tree-node [post=tree-maybe-post children=tree-internal-graph] - +$ tree-internal-graph - $~ [%empty ~] - $% [%graph p=tree-graph] - [%empty ~] - == - +$ tree-update-logs (tree [resource tree-update-log]) - +$ tree-update-log (tree [time tree-logged-update]) - +$ tree-logged-update - $: p=time - $= q - $% [%add-graph =resource =tree-graph mark=(unit ^mark) ow=?] - [%add-nodes =resource nodes=(tree [index tree-node])] - [%remove-posts =resource indices=(tree index)] - [%add-signatures =uid signatures=tree-signatures] - [%remove-signatures =uid signatures=tree-signatures] - == - == - +$ tree-signatures (tree signature) - +$ tree-maybe-post (each tree-post hash) - +$ tree-post - $: author=ship - =index - time-sent=time - contents=(list content) - hash=(unit hash) - signatures=tree-signatures - == - :: - ++ remake-network - |= t=tree-network - ^- network - :* (remake-graphs graphs.t) - (remake-jug:migrate tag-queries.t) - (remake-update-logs update-logs.t) - (remake-graphs archive.t) - ~ - == - :: - ++ remake-graphs - |= t=tree-graphs - ^- graphs - %- remake-map:migrate - (~(run by t) remake-marked-graph) - :: - ++ remake-marked-graph - |= t=tree-marked-graph - ^- marked-graph - [(remake-graph p.t) q.t] - :: - ++ remake-graph - |= t=tree-graph - ^- graph - %+ gas:orm *graph - %+ turn ~(tap by t) - |= [a=atom tn=tree-node] - ^- [atom node] - [a (remake-node tn)] - :: - ++ remake-internal-graph - |= t=tree-internal-graph - ^- internal-graph - ?: ?=(%empty -.t) - [%empty ~] - [%graph (remake-graph p.t)] - :: - ++ remake-node - |= t=tree-node - ^- node - :- (remake-post post.t) - (remake-internal-graph children.t) - :: - ++ remake-update-logs - |= t=tree-update-logs - ^- update-logs - %- remake-map:migrate - (~(run by t) remake-update-log) - :: - ++ remake-update-log - |= t=tree-update-log - ^- update-log - =/ ulm ((on time logged-update) gth) - %+ gas:ulm *update-log - %+ turn ~(tap by t) - |= [=time tlu=tree-logged-update] - ^- [^time logged-update] - [time (remake-logged-update tlu)] - :: - ++ remake-logged-update - |= t=tree-logged-update - ^- logged-update - :- p.t - ?- -.q.t - %add-graph - :* %add-graph - resource.q.t - (remake-graph tree-graph.q.t) - mark.q.t - ow.q.t - == - :: - %add-nodes - :- %add-nodes - :- resource.q.t - %- remake-map:migrate - (~(run by nodes.q.t) remake-node) - :: - %remove-posts - [%remove-posts resource.q.t (remake-set:migrate indices.q.t)] - :: - %add-signatures - [%add-signatures uid.q.t (remake-set:migrate signatures.q.t)] - :: - %remove-signatures - [%remove-signatures uid.q.t (remake-set:migrate signatures.q.t)] - == - :: - ++ remake-post - |= t=tree-maybe-post - ^- maybe-post - ?- -.t - %| t - %& t(signatures.p (remake-set:migrate signatures.p.t)) - == - :: - ++ try-rejoin - |= [rid=resource nack-count=@] - ^- card:agent:gall - =/ res-path (en-path:res rid) - =/ wire [%try-rejoin (scot %ud nack-count) res-path] - =/ =cage - :- %pull-hook-action - !> ^- action:pull-hook - [%add [entity .]:rid] - [%pass wire %agent [our %graph-pull-hook] %poke cage] - -- --- diff --git a/desk/sur/activity.hoon b/desk/sur/activity.hoon deleted file mode 100644 index 8f5d1f0f..00000000 --- a/desk/sur/activity.hoon +++ /dev/null @@ -1,368 +0,0 @@ -/- c=channels, t=contacts, ch=chat, g=groups -/+ mp=mop-extensions -|% -+| %collections -:: $stream: the activity stream comprised of events from various agents -+$ stream ((mop time event) lte) -:: $indices: the stream and its read data split into various indices -+$ indices - $~ [[[%base ~] *index] ~ ~] - (map source index) -:: $volume-settings: the volume settings for each source -+$ volume-settings (map source volume-map) -:: $activity: the current state of activity for each source -+$ activity (map source activity-summary) -:: $full-info: the full state of the activity stream -+$ full-info [=indices =activity =volume-settings] -:: $volume-map: how to badge and notify for each event type -+$ volume-map - $~ default-volumes - (map event-type volume) -:: $feed: a set of grouped events and the summaries of their sources -+$ feed - $: feed=(list activity-bundle) - summaries=activity - == -+$ feed-init - $: all=(list activity-bundle) - mentions=(list activity-bundle) - replies=(list activity-bundle) - summaries=activity - == -+| %actions -:: $action: how to interact with our activity stream -:: -:: actions are only ever performed for and by our selves -:: -:: %add: add an event to the stream -:: %bump: mark a source as having new activity from myself -:: %del: remove a source and all its activity -:: %read: mark an event as read -:: %adjust: adjust the volume of an source -:: %allow-notifications: change which notifications are allowed -:: -+$ action - $% [%add =incoming-event] - [%bump =source] - [%del =source] - [%del-event =source event=incoming-event] - [%read =source =read-action] - [%adjust =source =(unit volume-map)] - [%allow-notifications allow=notifications-allowed] - == -:: -:: $read-action: mark activity read -:: -:: %item: (DEPRECATED) mark an individual activity as read, indexed by id -:: %event: (DEPRECATED) mark an individual activity as read, indexed by the event itself -:: %all: mark _everything_ as read for this source, and possibly children -:: -+$ read-action - $% [%item id=time-id] - [%event event=incoming-event] - [%all time=(unit time) deep=?] - == -:: -+| %updates -:: -:: $update: what we hear after an action -:: -:: %add: an event was added to the stream -:: %del: a source and its activity were removed -:: %read: a source's activity state was updated -:: %activity: the activity state was updated -:: %adjust: the volume of a source was adjusted -:: %allow-notifications: the allowed notifications were changed -:: -+$ update - $% [%add =source time-event] - [%del =source] - [%read =source =activity-summary] - [%activity =activity] - [%adjust =source volume-map=(unit volume-map)] - [%allow-notifications allow=notifications-allowed] - == -:: -+| %basics -:: $event: a single point of activity, from one of our sources -:: -:: $incoming-event: the event that was sent to us -:: .notified: if this event has been notified -:: .child: if this event is from a child source -:: -+$ event - $: incoming-event - notified=? - child=? - == -+$ incoming-event - $% [%post post-event] - [%reply reply-event] - [%dm-invite =whom] - [%dm-post dm-post-event] - [%dm-reply dm-reply-event] - [%group-ask group=flag:g =ship] - [%group-kick group=flag:g =ship] - [%group-join group=flag:g =ship] - [%group-invite group=flag:g =ship] - [%chan-init channel=nest:c group=flag:g] - [%group-role group=flag:g =ship roles=(set sect:g)] - [%flag-post key=message-key channel=nest:c group=flag:g] - [%flag-reply key=message-key parent=message-key channel=nest:c group=flag:g] - [%contact contact-event] - == -:: -+$ post-event - $: key=message-key - channel=nest:c - group=flag:g - content=story:c - mention=? - == -:: -+$ reply-event - $: key=message-key - parent=message-key - channel=nest:c - group=flag:g - content=story:c - mention=? - == -:: -+$ dm-post-event - $: key=message-key - =whom - content=story:c - mention=? - == -:: -+$ dm-reply-event - $: key=message-key - parent=message-key - =whom - content=story:c - mention=? - == -:: -+$ contact-event - $: who=ship - update=(pair @tas value:t) - == -:: -:: $source: where the activity is happening -+$ source - $% [%base ~] - [%group =flag:g] - [%channel =nest:c group=flag:g] - [%thread key=message-key channel=nest:c group=flag:g] - [%dm =whom] - [%dm-thread key=message-key =whom] - [%contact who=ship] - == -:: -:: $index: the stream of activity and read state for a source -+$ index [=stream =reads bump=time] -:: -:: $reads: the read state for a source -:: -:: $floor: the time of the latest event that was read -:: $items: the set of events above the floor that have been read -:: -+$ reads - $: floor=time - items=read-items - == -+$ read-items ((mop time-id ,~) lte) -:: $activity-summary: the summary of activity for a source -:: -:: $newest: the time of the latest activity read or unread -:: $count: the total number of unread events including children -:: $notify-count: the number of unreads that are notifications -:: including children -:: $notify: if there are any notifications here or in children -:: $unread: if the main stream of source is unread: which starting -:: message, how many there are, and if any are notifications -:: $children: the sources nested under this source -:: -+$ activity-summary - $~ [*@da 0 0 | ~ ~ ~] - $: newest=time - count=@ud - notify-count=@ud - notify=_| - unread=(unit unread-point) - children=(set source) - reads=* :: DO NOT USE, 🚨 ⚠️ REMOVE - == -+$ unread-point [message-key count=@ud notify=_|] -+$ volume [unreads=? notify=?] -+$ notifications-allowed ?(%all %some %none) -+$ activity-bundle - $: =source - latest=time - events=(list time-event) - == -:: -+| %primitives -+$ whom - $% [%ship p=ship] - [%club p=id:club:ch] - == -+$ time-id time -+$ message-id (pair ship time-id) -+$ message-key [id=message-id =time] -:: -+$ event-type - $? %chan-init - %post - %post-mention - %reply - %reply-mention - %dm-invite - %dm-post - %dm-post-mention - %dm-reply - %dm-reply-mention - %group-invite - %group-kick - %group-join - %group-ask - %group-role - %flag-post - %flag-reply - %contact - == -+| %helpers -+$ time-event [=time =event] -++ on-event ((on time event) lte) -++ ex-event ((mp time event) lte) -++ on-read-items ((on time ,~) lte) -+| %constants -++ default-volumes - ^~ - ^- (map event-type volume) - %- my - :~ [%post & &] - [%reply & |] - [%dm-reply & &] - [%post-mention & &] - [%reply-mention & &] - [%dm-invite & &] - [%dm-post & &] - [%dm-post-mention & &] - [%dm-reply-mention & &] - [%group-invite & &] - [%group-ask & &] - [%flag-post & &] - [%flag-reply & &] - [%group-kick & |] - [%group-join & |] - [%group-role & |] - ::XX remove notify? - [%contact & &] - == -++ old-volumes - ^~ - %- my - :~ [%soft (~(put by default-volumes) %post [& |])] - [%loud (~(run by default-volumes) |=([u=? *] [u &]))] - [%hush (~(run by default-volumes) |=([u=? *] [u |]))] - == -++ mute - ^~ - (~(run by default-volumes) |=(* [| |])) -:: -+| %old-types -++ old - |% - ++ v7 - |% - +$ stream ((mop time event) lte) - +$ event - $: incoming-event - notified=? - child=? - == - +$ incoming-event - $% [%post post-event] - [%reply reply-event] - [%dm-invite =whom] - [%dm-post dm-post-event] - [%dm-reply dm-reply-event] - [%group-ask group=flag:g =ship] - [%group-kick group=flag:g =ship] - [%group-join group=flag:g =ship] - [%group-invite group=flag:g =ship] - [%chan-init channel=nest:c group=flag:g] - [%group-role group=flag:g =ship roles=(set sect:g)] - [%flag-post key=message-key channel=nest:c group=flag:g] - [%flag-reply key=message-key parent=message-key channel=nest:c group=flag:g] - == - +$ source - $% [%base ~] - [%group =flag:g] - [%channel =nest:c group=flag:g] - [%thread key=message-key channel=nest:c group=flag:g] - [%dm =whom] - [%dm-thread key=message-key =whom] - == - +$ index [=stream =reads bump=time] - -- - ++ v4 - |% - +$ feed (list activity-bundle) - -- - ++ v3 - |% - +$ index [=stream =reads] - +$ indices (map source index) - +$ update - $% [%add =source time-event] - [%del =source] - [%read =source =activity-summary] - [%adjust =source volume-map=(unit volume-map)] - [%allow-notifications allow=notifications-allowed] - == - +$ full-info - $: =indices - =activity - =volume-settings - == - +$ activity (map source activity-summary) - +$ activity-summary - $~ [*@da 0 0 | ~ ~ [*@da ~]] - $: newest=time - count=@ud - notify-count=@ud - notify=_| - unread=(unit unread-point) - children=(unit activity) - =reads - == - -- - ++ v2 - |% - +$ update - $% [%add =source time-event] - [%del =source] - [%read =source =activity-summary] - [%adjust =source volume-map=(unit volume-map)] - [%allow-notifications allow=notifications-allowed] - == - +$ full-info - $: =indices:v3 - activity=activity - =volume-settings - == - +$ activity (map source activity-summary) - +$ activity-summary - $~ [*@da 0 | ~ ~] - $: newest=time - count=@ud - notify=_| - unread=(unit unread-point) - children=(unit activity) - == - -- - -- --- diff --git a/peru.yaml b/peru.yaml new file mode 100644 index 00000000..0846470c --- /dev/null +++ b/peru.yaml @@ -0,0 +1,60 @@ +# peru.yaml — vendor desk dependencies from upstream Urbit. +# +# Run `./scripts/sync-deps.sh` (or `peru sync`) to populate desk-deps/ with the +# files picked below. desk-deps/ is an entirely-gitignored vendored tree kept +# SEPARATE from our own source in desk/. A desk is assembled by copying +# desk-deps/ in first (with --delete) and then our desk/ on top — see +# scripts/assemble-desk.sh. +# +# Why peru instead of rsyncing all of pkg/base-dev: base-dev ships ~92 files +# (bitcoin/eth-adjacent, language-server, wasm/umd, woff2, png, ...). The new +# kernel builds EVERY mark present on a desk, so copying the whole tree in pulls +# files this desk does not use, some of which fail to compile on the target +# kelvin. We pick only the transitively-required subset (computed from the +# import closure of desk/). +# +# IMPORTANT: do NOT add anything this repo owns to the pick lists. Everything +# landscape owns (docket, hark, treaty, settings, storage, the *-json libs, the +# glob/svg/charge marks, ...) already lives committed in desk/. + +imports: + base: desk-deps/ + +# %base (urbit/urbit) dev dependencies. Pinned to the 408 kelvin tag, matching +# desk/sys.kelvin ([%zuse 408]). To build against a different kernel, change +# `rev` here on your branch. +git module base: + url: https://github.com/urbit/urbit.git + rev: 408k-rc2 + pick: + # libraries the desk imports (transitive closure of desk/): + # agents -> default-agent/dbug/verb/agentio + # default-agent -> skeleton; server/mip standalone + # threads (ted/) -> strandio -> strand (+ sur/spider) + # lib/test-agent -> test + - pkg/base-dev/lib/agentio.hoon + - pkg/base-dev/lib/dbug.hoon + - pkg/base-dev/lib/default-agent.hoon + - pkg/base-dev/lib/mip.hoon + - pkg/base-dev/lib/server.hoon + - pkg/base-dev/lib/skeleton.hoon + - pkg/base-dev/lib/strand.hoon + - pkg/base-dev/lib/strandio.hoon + - pkg/base-dev/lib/test.hoon + - pkg/base-dev/lib/verb.hoon + # thread types (spider sur) + verb sur (pulled in by lib/verb) + - pkg/base-dev/sur/spider.hoon + - pkg/base-dev/sur/verb.hoon + # standard marks clay needs to build/serve the desk, plus the marks our own + # marks convert through (grad/grow/grab over %noun, %mime, %json) + - pkg/base-dev/mar/bill.hoon + - pkg/base-dev/mar/hoon.hoon + - pkg/base-dev/mar/kelvin.hoon + - pkg/base-dev/mar/mime.hoon + - pkg/base-dev/mar/noun.hoon + - pkg/base-dev/mar/txt.hoon + - pkg/base-dev/mar/json.hoon + - pkg/base-dev/mar/css.hoon + - pkg/base-dev/mar/htm.hoon + - pkg/base-dev/mar/html.hoon + export: pkg/base-dev diff --git a/scripts/assemble-desk.sh b/scripts/assemble-desk.sh new file mode 100755 index 00000000..ebbe1839 --- /dev/null +++ b/scripts/assemble-desk.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# scripts/assemble-desk.sh +# +# Assemble a complete %landscape desk into by layering: +# 1. desk-deps/ (peru-vendored upstream deps) -- rsync'd in with --delete +# 2. desk/ (our own source) -- rsync'd on top +# +# Step 1's --delete clears any stale files already in the target (e.g. the old +# full base-dev a ship was built with, or files we've since removed). Step 2 +# overlays our source, which wins on any overlap. The result is exactly +# desk-deps/ + desk/. +# +# By default this runs `peru sync` first to ensure desk-deps/ is populated and +# current. Set SKIP_SYNC=true to skip that (e.g. when syncing once before +# assembling into many targets). + +set -euo pipefail +cd "$(dirname "$0")/.." + +target="${1:-}" +if [ -z "$target" ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +if [ "${SKIP_SYNC:-false}" != "true" ]; then + ./scripts/sync-deps.sh +fi + +if [ ! -d desk-deps ]; then + echo "desk-deps/ not found — run scripts/sync-deps.sh (peru sync) first" >&2 + exit 1 +fi + +mkdir -p "$target" +# 1. vendored deps first, clearing anything stale in the target +rsync -aL --delete desk-deps/ "$target/" +# 2. our own source on top (wins on overlap) +rsync -aL desk/ "$target/" + +# stamp the build commit, like the deploy pipeline does +git rev-parse --short HEAD > "$target/commit.txt" 2>/dev/null || true + +echo "Assembled desk into $target" diff --git a/scripts/sync-deps.sh b/scripts/sync-deps.sh new file mode 100755 index 00000000..e8591d18 --- /dev/null +++ b/scripts/sync-deps.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# scripts/sync-deps.sh +# Vendor %base (urbit/urbit) desk dependencies into desk-deps/ using peru, per +# peru.yaml. Run this after cloning, and after changing peru.yaml. Assemble a +# desk from desk-deps/ + desk/ with scripts/assemble-desk.sh. + +set -euo pipefail +cd "$(dirname "$0")/.." + +if ! command -v peru >/dev/null 2>&1; then + cat >&2 <<'EOF' +peru is not installed. Install it: + pipx install peru # recommended + pip install peru # via pip + brew install peru # via homebrew +See https://github.com/buildinspace/peru +EOF + exit 1 +fi + +peru sync +echo "Desk dependencies synced." From 7e0cf1926738bc8eda850b9fc2d6d8b6d71f6698 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Thu, 18 Jun 2026 13:36:27 -0500 Subject: [PATCH 2/3] deploy: enable NumPy for gcloud IAP tunnel to speed up the scp upload The assembled desk tarball is shipped up through gcloud's single-threaded IAP websocket proxy; without NumPy that upload takes ~4 min. Matches the tlon-apps deploy.sh fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/helpers/deploy.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/helpers/deploy.sh b/.github/helpers/deploy.sh index ebf5b803..93b33c39 100755 --- a/.github/helpers/deploy.sh +++ b/.github/helpers/deploy.sh @@ -43,6 +43,20 @@ fi # Package the assembled, self-contained desk. tar czf "$workdir/desk.tgz" -C "$workdir" assembled +# --- Speed up the IAP tunnel ------------------------------------------------ +# gcloud's IAP TCP forwarding is a single-threaded Python websocket proxy. With +# NumPy available its upload path is dramatically faster (gcloud prints a +# warning recommending this). The whole assembled desk is shipped UP through +# this tunnel, so without NumPy the scp below takes ~4 min. CLOUDSDK_PYTHON_- +# SITEPACKAGES lets gcloud's bundled interpreter see the system-installed numpy. +export CLOUDSDK_PYTHON_SITEPACKAGES=1 +if ! python3 -c 'import numpy' >/dev/null 2>&1; then + echo "Installing NumPy to speed up the IAP tunnel..." + pip3 install --user numpy \ + || pip3 install --user --break-system-packages numpy \ + || echo "WARNING: NumPy install failed; tunnel upload will be slow." +fi + # --- SSH key setup ---------------------------------------------------------- sshpriv=$(mktemp "${TMPDIR:-/tmp/}ssh.XXXXXXXXX") sshpub=$sshpriv.pub From 4018d93617ad6dc228fe6829bb3c589063c4335c Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Thu, 18 Jun 2026 13:53:16 -0500 Subject: [PATCH 3/3] deploy: unmount desk before mounting Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/helpers/deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/helpers/deploy.sh b/.github/helpers/deploy.sh index 93b33c39..d56dc09f 100755 --- a/.github/helpers/deploy.sh +++ b/.github/helpers/deploy.sh @@ -81,6 +81,7 @@ cat > "$cmdfile" <