From 57b107b194b0682ae55653389ef59970454fce2e Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 12:48:54 -0400 Subject: [PATCH 01/19] feat(argo): add ChairLift Wayland behave lane run-chairlift-wayland-tests builds a ChairLift branch, boots run-container-tests' nested systemd/GDM Dakota target with a headless GNOME Shell, and runs ChairLift's behave AT-SPI suite inside the live Wayland session through qecore-headless. On x86_64 it then installs Enhanced Troubleshooting's packages through the target's Homebrew and checks that Goose desktop launched in ChairLift's profile writes nothing outside it. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 865 ++++++++++++++++++ .../unit/test_run_chairlift_wayland_tests.py | 141 +++ 2 files changed, 1006 insertions(+) create mode 100644 argo/workflow-templates/run-chairlift-wayland-tests.yaml create mode 100644 tests/unit/test_run_chairlift_wayland_tests.py diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml new file mode 100644 index 0000000000..baec6916e0 --- /dev/null +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -0,0 +1,865 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: run-chairlift-wayland-tests + namespace: argo + annotations: + description: | + Builds ChairLift from a branch and runs its behave + dogtail AT-SPI suite + (test/e2e/features) inside a live GNOME Wayland session: the nested, + systemd-booted Dakota target run-container-tests boots (headless GNOME + Shell with a virtual monitor, GDM autologin, test-user linger), driven by + qecore-headless --session-type wayland. No X server is started; the + application runs with GDK_BACKEND=wayland and DISPLAY unset. On x86_64 it + then installs Enhanced Troubleshooting's packages through the target's + Homebrew and checks that Goose desktop launched in ChairLift's profile + writes nothing outside that profile. + labels: + app.kubernetes.io/component: test-runner + app.kubernetes.io/part-of: bluefin-test-suite +spec: + serviceAccountName: argo + entrypoint: run-chairlift-wayland-tests + podGC: + strategy: OnWorkflowSuccess + ttlStrategy: + secondsAfterSuccess: 604800 + secondsAfterFailure: 2592000 + arguments: + parameters: + - name: chairlift-repo + value: "https://github.com/projectbluefin/chairlift" + - name: chairlift-branch + value: "main" + - name: image + value: "ghcr.io/projectbluefin/dakota" + - name: image-tag + value: "testing" + - name: behave-tags + value: "" + - name: goose-isolation-check + value: "true" + templates: + - name: run-chairlift-wayland-tests + # Same placement as run-container-tests: ghost is the only node with a + # graphical seat and the canonical local-path result store. + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + kubernetes.io/hostname: ghost + # Build (~5m) + pull (≤34m worst case, normally minutes) + boot and pip + # (~5m) + one application launch per scenario (~25m, the suite's own + # budget) + the Goose check's network installs (~10m). + activeDeadlineSeconds: 5400 + # Leaf template holding the shared ghost QA capacity key, like + # run-container-tests: each run boots one nested GNOME target. + synchronization: + semaphores: + - configMapKeyRef: + name: workflow-semaphores + key: ghost-container-qa + inputs: + parameters: + - name: chairlift-repo + value: "{{workflow.parameters.chairlift-repo}}" + - name: chairlift-branch + value: "{{workflow.parameters.chairlift-branch}}" + - name: image + value: "{{workflow.parameters.image}}" + - name: image-tag + value: "{{workflow.parameters.image-tag}}" + - name: behave-tags + value: "{{workflow.parameters.behave-tags}}" + - name: goose-isolation-check + value: "{{workflow.parameters.goose-isolation-check}}" + outputs: + parameters: + - name: result + valueFrom: + path: /tmp/results/result-summary.txt + default: "No results generated" + - name: failed-scenarios + valueFrom: + path: /tmp/results/failed-scenarios.json + default: "[]" + - name: goose-isolation + valueFrom: + path: /tmp/results/goose/verdict.json + default: "{}" + initContainers: + # The Go build happens in the pod, before the target exists, so the + # suite runs the branch's own chairlift_e2e binary and nothing else. + # Schemas are compiled later inside the Dakota target, whose + # glib-compile-schemas is the one the application will read them with; + # `make schemas` skips cleanly here because the builder lacks it. + - name: build-chairlift + image: golang:1.26 + resources: + requests: + cpu: "2" + memory: 2Gi + limits: + cpu: "4" + memory: 4Gi + volumeMounts: + - mountPath: /workspace + name: workspace + env: + - name: CHAIRLIFT_REPO + value: "{{inputs.parameters.chairlift-repo}}" + - name: CHAIRLIFT_BRANCH + value: "{{inputs.parameters.chairlift-branch}}" + command: [bash, -c] + args: + - | + set -euo pipefail + # A missing branch must fail: never report results from other code. + git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}" /workspace/chairlift + cd /workspace/chairlift + mkdir -p /workspace/lane/bin /workspace/lane/inventory + git rev-parse HEAD | tee /workspace/lane/chairlift-commit + CGO_ENABLED=0 make build-e2e schemas + + # Lane tools live inside the module so they may import internal/. + # navjson: the page and shortcut inventories the Go gate + # (test/e2e/atspi_behave_test.go) hands the suite, from the one + # authority, internal/navigation. + mkdir -p build/lane/navjson build/lane/gooseprofile + cat >build/lane/navjson/main.go <<'GO' + package main + + import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/projectbluefin/chairlift/internal/branding" + "github.com/projectbluefin/chairlift/internal/navigation" + ) + + type item struct { + Name string `json:"name"` + Title string `json:"title"` + } + + func write(dir, name string, value any) { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0o644); err != nil { + panic(err) + } + } + + func main() { + dir := os.Args[1] + items := navigation.Items() + nav := make([]item, 0, len(items)) + for _, it := range items { + nav = append(nav, item{Name: it.Name, Title: it.Title}) + } + var shortcuts []string + for _, s := range navigation.Shortcuts(items) { + shortcuts = append(shortcuts, s.Title) + } + write(dir, "navigation.json", nav) + write(dir, "shortcuts.json", shortcuts) + if err := os.WriteFile(filepath.Join(dir, "app-name.txt"), []byte(branding.AppName), 0o644); err != nil { + panic(err) + } + } + GO + # gooseprofile: writes Enhanced Troubleshooting's Goose profile with + # the branch's own internal/troubleshoot, and reads back which + # extensions a config.yaml leaves enabled. + cat >build/lane/gooseprofile/main.go <<'GO' + package main + + import ( + "fmt" + "os" + "sort" + + "github.com/projectbluefin/chairlift/internal/troubleshoot" + "gopkg.in/yaml.v3" + ) + + func main() { + switch { + case len(os.Args) == 4 && os.Args[1] == "write": + profile := troubleshoot.ProfileAt(os.Args[2]) + if err := profile.Write(os.Args[3]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(profile.ConfigPath()) + case len(os.Args) == 3 && os.Args[1] == "enabled": + data, err := os.ReadFile(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + var cfg struct { + Extensions map[string]struct { + Enabled bool `yaml:"enabled"` + } `yaml:"extensions"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + var names []string + for name, ext := range cfg.Extensions { + if ext.Enabled { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + fmt.Println(name) + } + default: + fmt.Fprintln(os.Stderr, "usage: gooseprofile write | enabled ") + os.Exit(2) + } + } + GO + CGO_ENABLED=0 go build -o /workspace/lane/bin/navjson ./build/lane/navjson + CGO_ENABLED=0 go build -o /workspace/lane/bin/gooseprofile ./build/lane/gooseprofile + /workspace/lane/bin/navjson /workspace/lane/inventory + cat /workspace/lane/inventory/navigation.json; echo + chmod -R a+rX /workspace + ls -l build/e2e/chairlift /workspace/lane/bin + script: + image: ghcr.io/projectbluefin/arc-runner:latest + imagePullPolicy: IfNotPresent + securityContext: + # Podman needs a writable cgroup hierarchy to boot the nested systemd host. + privileged: true + runAsUser: 0 + allowPrivilegeEscalation: true + resources: + requests: + cpu: "2" + memory: 4Gi + ephemeral-storage: 12Gi + limits: + cpu: "4" + memory: 8Gi + ephemeral-storage: 24Gi + env: + - name: IMAGE_REPO + value: "{{inputs.parameters.image}}" + - name: IMAGE_TAG + value: "{{inputs.parameters.image-tag}}" + - name: BEHAVE_TAGS + value: "{{inputs.parameters.behave-tags}}" + - name: GOOSE_CHECK + value: "{{inputs.parameters.goose-isolation-check}}" + volumeMounts: + - mountPath: /workspace + name: workspace + - mountPath: /tmp/results + name: results + - mountPath: /var/lib/containers + name: containers-storage + - mountPath: /run/containers + name: containers-runroot + - mountPath: /run/udev + name: udev + - mountPath: /var/mnt/ghost-data/test-results + name: results-store + - mountPath: /var/cache/bluefin-qa-pip + name: pip-cache + command: [bash] + source: | + set -euo pipefail + TARGET_NAME="chairlift-wayland-target" + RESULT_DIR="/var/mnt/ghost-data/test-results/{{workflow.name}}/chairlift-wayland" + mkdir -p /tmp/results "${RESULT_DIR}" /opt/qa-wheels /var/cache/bluefin-qa-pip + + persist_results() { + mkdir -p "${RESULT_DIR}" + cp -a /tmp/results/. "${RESULT_DIR}/" || { + echo "ERROR: failed to persist results to ${RESULT_DIR}" >&2 + return 1 + } + touch "${RESULT_DIR}/.complete" + } + cleanup_target() { + podman rm --force "${TARGET_NAME}" >/dev/null 2>&1 || true + } + on_exit() { + local rc=$? + local persist_rc=0 + persist_results || persist_rc=$? + cleanup_target + if [[ "${persist_rc}" -ne 0 && "${rc}" -eq 0 ]]; then + rc="${persist_rc}" + fi + return "${rc}" + } + trap on_exit EXIT + # activeDeadlineSeconds and `argo stop` arrive as signals; an untrapped + # SIGTERM would skip the EXIT trap and leave the privileged target up. + trap 'exit 143' TERM + trap 'exit 130' INT + + command -v podman >/dev/null || { + echo "Podman is required for nested systemd QA" >&2 + exit 1 + } + echo "ChairLift commit: $(cat /workspace/lane/chairlift-commit)" | tee /tmp/results/chairlift-commit.txt >&2 + + # --- target image: through the local Zot pull-through cache -------- + zot_image() { + case "$1" in + ghcr.io/*) printf '192.168.1.102:30501/ghcr/%s' "${1#ghcr.io/}" ;; + quay.io/*) printf '192.168.1.102:30501/quay/%s' "${1#quay.io/}" ;; + docker.io/*) printf '192.168.1.102:30501/docker/%s' "${1#docker.io/}" ;; + *) printf '%s' "$1" ;; + esac + } + ZOT_IMAGE_REPO=$(zot_image "${IMAGE_REPO}") + PULL_TLS_ARGS=() + case "${ZOT_IMAGE_REPO}" in + 192.168.1.102:*|localhost:*|127.0.0.1:*) + mkdir -p /etc/containers/registries.conf.d + printf '%s\n' '[[registry]]' 'location = "192.168.1.102:30501"' 'insecure = true' \ + >/etc/containers/registries.conf.d/bluefin-local-zot.conf + PULL_TLS_ARGS=(--tls-verify=false) + ;; + esac + DIGEST=$(timeout 120 skopeo inspect "${PULL_TLS_ARGS[@]}" --no-tags \ + --format '{{.Digest}}' "docker://${ZOT_IMAGE_REPO}:${IMAGE_TAG}" 2>/dev/null || true) + if [[ -n "${DIGEST}" ]]; then + TARGET_IMAGE="${ZOT_IMAGE_REPO}@${DIGEST}" + else + echo "Warning: could not resolve a digest for ${IMAGE_REPO}:${IMAGE_TAG}; using the tag." >&2 + TARGET_IMAGE="${ZOT_IMAGE_REPO}:${IMAGE_TAG}" + fi + echo "${IMAGE_REPO}:${IMAGE_TAG} ${DIGEST:-unresolved}" | tee /tmp/results/target-image.txt >&2 + cat >/tmp/storage.conf <<'EOF' + [storage] + driver = "overlay" + graphroot = "/var/lib/containers/storage" + runroot = "/run/containers/storage" + EOF + export CONTAINERS_STORAGE_CONF=/tmp/storage.conf + # Bounded attempts: the Zot cache occasionally stalls mid-blob, and + # podman keeps finished blobs across attempts (see run-container-tests). + pull_ok=false + for attempt in 1 2 3 4; do + echo "Pulling ${TARGET_IMAGE} (attempt ${attempt}/4)..." >&2 + if timeout 480 podman pull "${PULL_TLS_ARGS[@]}" "${TARGET_IMAGE}" >/dev/null; then + pull_ok=true + break + fi + sleep 30 + done + [[ "${pull_ok}" == true ]] || { echo "Giving up on ${TARGET_IMAGE}" >&2; exit 1; } + + # --- boot the nested systemd/GDM target ---------------------------- + # /usr/share/chairlift and /etc/chairlift are masked: a shipped + # config.yml outranks every configuration fixture, and the suite's + # before_all refuses to run against it. /proc/cmdline is blanked so a + # composefs host's command line cannot send the real bootc reader to + # host state (the same isolation test/e2e/dakota_atspi.sh applies). + podman run --detach --systemd=always --network host --privileged \ + --cgroupns=host \ + --security-opt seccomp=unconfined \ + --tmpfs /run:rw,nosuid,nodev,mode=755,size=256m \ + --tmpfs /usr/share/chairlift:ro,notmpcopyup \ + --tmpfs /etc/chairlift:ro,notmpcopyup \ + --mount type=bind,source=/dev/null,destination=/proc/cmdline,ro \ + --name "${TARGET_NAME}" \ + --pull=never \ + --mount type=bind,source=/sys,destination=/sys,ro,bind-propagation=rprivate \ + --mount type=bind,source=/run/udev,destination=/run/udev,ro,bind-propagation=rprivate \ + --volume /workspace:/workspace:ro \ + --volume /tmp/results:/tmp/results \ + --volume /opt/qa-wheels:/var/opt/qa-wheels:ro \ + --volume /var/cache/bluefin-qa-pip:/var/cache/bluefin-qa-pip:rw \ + --volume /etc/resolv.conf:/etc/resolv.conf:ro \ + "${TARGET_IMAGE}" /sbin/init >/dev/null + + ready=false + for i in $(seq 1 60); do + state=$(podman exec "${TARGET_NAME}" systemctl is-system-running 2>/dev/null || true) + dbus_active=$(podman exec "${TARGET_NAME}" systemctl is-active dbus 2>/dev/null || true) + logind_active=$(podman exec "${TARGET_NAME}" systemctl is-active systemd-logind 2>/dev/null || true) + can_graphical=$(podman exec "${TARGET_NAME}" loginctl show-seat seat0 --property=CanGraphical 2>/dev/null || true) + echo "systemd readiness probe ${i}: state=${state:-unknown} dbus=${dbus_active:-unknown} logind=${logind_active:-unknown} seat0=${can_graphical:-unknown}" + if [[ "${state}" == "running" || "${state}" == "degraded" ]] \ + && [[ "${dbus_active}" == "active" ]] \ + && [[ "${logind_active}" == "active" ]] \ + && [[ "${can_graphical}" == "CanGraphical=yes" ]]; then + ready=true + break + fi + sleep 2 + done + if [[ "${ready}" != true ]]; then + echo "Nested systemd host did not reach a usable session state" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -n 300 || true + exit 1 + fi + + echo "Provisioning nested target..." >&2 + # Fed over stdin through a quoted heredoc, never `bash -c '...'` + # (see run-container-tests: an inner apostrophe truncates the latter). + setup_rc=0 + podman exec -i --env GOOSE_CHECK="${GOOSE_CHECK}" "${TARGET_NAME}" bash -s <<'NESTED_SETUP' || setup_rc=$? + set -euo pipefail + mkdir -p /run/dbus /run/systemd/seats /run/systemd/users + systemctl stop gdm || true + rm -rf /run/gdm /var/log/gdm + mkdir -p /run/gdm /var/log/gdm + getent group gdm >/dev/null 2>&1 || groupadd -r gdm + id -u gdm >/dev/null 2>&1 || useradd -r -g gdm -d /var/lib/gdm -s /usr/sbin/nologin gdm + chown -R gdm:gdm /run/gdm /var/log/gdm + chmod 755 /run/gdm + chmod 750 /var/log/gdm + + # Local group entries for the device groups; NSS-only groups are + # materialised because useradd accepts only /etc/group entries. + append_group_line() { + if [ -s /etc/group ] && [ "$(tail -c 1 /etc/group | wc -l)" -eq 0 ]; then + printf '\n' >>/etc/group + fi + printf '%s\n' "$1" >>/etc/group + } + for group in video render input; do + if ! grep -q "^${group}:" /etc/group; then + if ! groupadd -r "${group}" 2>/dev/null; then + nss_entry=$(getent group "${group}" || true) + if [ -n "${nss_entry}" ]; then + append_group_line "${nss_entry}" + else + case "${group}" in video) gid=39 ;; render) gid=989 ;; *) gid=999 ;; esac + append_group_line "${group}:x:${gid}:" + fi + fi + fi + grep -q "^${group}:" /etc/group || { echo "Failed to create local group ${group}" >&2; exit 1; } + done + if ! id bluefin-test >/dev/null 2>&1; then + useradd -m -u 1000 -G video,render,input bluefin-test + else + usermod -aG video,render,input bluefin-test + fi + # Synthetic input: the container's own /dev/uinput, given to the + # lane-local input group (see run-container-tests). + [ -e /dev/uinput ] || modprobe uinput 2>/dev/null || true + if [ -e /dev/uinput ]; then + chgrp input /dev/uinput && chmod 0660 /dev/uinput + else + echo "WARNING: /dev/uinput is absent" >&2 + fi + # GDM PAM autologin needs a non-expired shadow lastchg. + if command -v chage >/dev/null 2>&1; then + chage -d "$(date +%Y-%m-%d)" bluefin-test + else + today=$(( $(date +%s) / 86400 )) + sed -i "s/^bluefin-test:!*/bluefin-test:!:${today}:0:99999:7:::/" /etc/shadow + fi + mkdir -p /etc/sudoers.d + echo "bluefin-test ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/bluefin-test + chmod 0440 /etc/sudoers.d/bluefin-test + mkdir -p /etc/gdm + printf "%s\n" "[daemon]" "AutomaticLoginEnable=True" "AutomaticLogin=bluefin-test" \ + "InitialSetupEnable=False" >/etc/gdm/custom.conf + mkdir -p /tmp/results/atspi /tmp/results/goose + chown -R 1000:1000 /home/bluefin-test /tmp/results + + # Headless shell, never DRM master; --unsafe-mode must be repeated + # because this ExecStart= reset overrides qecore's own edit. + mkdir -p /etc/systemd/user/org.gnome.Shell@.service.d + printf "%s\n" "[Service]" "ExecStart=" \ + "ExecStart=/usr/bin/gnome-shell --mode=%i --unsafe-mode --headless --virtual-monitor 1920x1080" \ + >/etc/systemd/user/org.gnome.Shell@.service.d/10-headless.conf + + # Dakota's session pulls in brew-preinstall.service (a user unit + # that installs the preinstall.d Brewfiles over the network), and + # its brew-update/upgrade timers run against the same prefix. None + # is under test, and each would contend with the Goose check's + # installs for Homebrew's locks. Masked before any user manager + # starts, since a manager reads unit files only at start. + systemctl --global mask brew-preinstall.service >/dev/null 2>&1 || true + systemctl mask --runtime brew-update.timer brew-upgrade.timer >/dev/null 2>&1 || true + + loginctl enable-linger bluefin-test + for _ in $(seq 1 30); do [ -S /run/user/1000/bus ] && break; sleep 1; done + if [ ! -S /run/user/1000/bus ]; then + echo "user@1000.service did not create /run/user/1000/bus after enable-linger" >&2 + loginctl show-user bluefin-test >&2 || true + systemctl status user@1000.service --no-pager --full >&2 || true + exit 1 + fi + + # qecore-headless (system interpreter; the target is disposable). + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --default-pip >&2 + export PIP_CACHE_DIR=/var/cache/bluefin-qa-pip + pip_args=(--disable-pip-version-check --no-input --cache-dir "${PIP_CACHE_DIR}") + if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_args+=(--break-system-packages) + fi + # setuptools<81: qecore imports pkg_resources (see run-container-tests). + qa_dependencies=("setuptools<81" qecore) + pip_ok=false + for attempt in 1 2 3; do + if python3 -m pip install "${pip_args[@]}" --find-links /var/opt/qa-wheels \ + "${qa_dependencies[@]}" >/tmp/pip-install.log 2>&1; then + pip_ok=true + break + fi + echo "pip install attempt ${attempt}/3 failed:" >&2 + tail -n 40 /tmp/pip-install.log >&2 + sleep 10 + done + [ "${pip_ok}" = true ] || { cat /tmp/pip-install.log >&2; exit 1; } + # python-uinput is qecore's uinput probe; Wayland input goes through + # gnome-ponytail-daemon, so a failed build here is a warning only. + python3 -m pip install "${pip_args[@]}" --find-links /var/opt/qa-wheels python-uinput \ + >/tmp/pip-uinput.log 2>&1 || { echo "WARNING: python-uinput did not install" >&2; tail -n 20 /tmp/pip-uinput.log >&2; } + # qecore-headless dconf ownership TOCTOU (see run-container-tests). + headless_script=$(command -v qecore-headless || true) + [ -n "${headless_script}" ] || { echo "qecore-headless not found" >&2; exit 1; } + if ! grep -q 'dconf/user || true"' "${headless_script}"; then + matches=$(grep -c '/dconf/user"$' "${headless_script}" || true) + [ "${matches}" = "1" ] || { echo "Unexpected qecore-headless layout (${matches})" >&2; exit 1; } + sed -i 's#/dconf/user"$#/dconf/user || true"#' "${headless_script}" + fi + python3 -c "import ast,sys; ast.parse(open(sys.argv[1]).read())" "${headless_script}" + + # The suite's own interpreter: a venv over the system site-packages + # (PyGObject, pyatspi, ponytail come from the image) with ChairLift's + # hash-pinned behave and dogtail, exactly as dakota_atspi.sh builds it. + python3 -m venv --system-site-packages /home/bluefin-test/atspi-venv + /home/bluefin-test/atspi-venv/bin/python -m pip install -q --disable-pip-version-check \ + --require-hashes -r /workspace/chairlift/test/e2e/requirements-atspi.txt + # Compile the branch's GSettings schemas with the image's own tool. + mkdir -p /home/bluefin-test/chairlift-schemas + cp /workspace/chairlift/data/*.gschema.xml /home/bluefin-test/chairlift-schemas/ + glib-compile-schemas /home/bluefin-test/chairlift-schemas + chown -R 1000:1000 /home/bluefin-test + + # The Goose check installs through the image's Homebrew, which + # brew-setup.service extracts on first boot. Assert the binary, not + # the unit's exit code (a satisfied Condition* also exits 0). + if [ "${GOOSE_CHECK}" = "true" ] && [ "$(uname -m)" = "x86_64" ]; then + brew_bin="" + for candidate in /home/linuxbrew/.linuxbrew/bin/brew /var/home/linuxbrew/.linuxbrew/bin/brew; do + [ -x "${candidate}" ] && brew_bin="${candidate}" && break + done + if [ -z "${brew_bin}" ]; then + systemctl unmask --runtime brew-setup.service || true + brew_setup_rc=0 + systemctl start brew-setup.service || brew_setup_rc=$? + for candidate in /home/linuxbrew/.linuxbrew/bin/brew /var/home/linuxbrew/.linuxbrew/bin/brew; do + [ -x "${candidate}" ] && brew_bin="${candidate}" && break + done + if [ -z "${brew_bin}" ]; then + echo "No Homebrew after brew-setup.service (start exit ${brew_setup_rc})" >&2 + systemctl status brew-setup.service --no-pager --full >&2 || true + exit 1 + fi + fi + echo "${brew_bin}" >/tmp/results/goose/brew-path.txt + ls -ld "$(dirname "$(dirname "$(dirname "${brew_bin}")")")" "$(dirname "$(dirname "${brew_bin}")")" >&2 + fi + + systemctl start gdm || { systemctl status gdm --no-pager --full >&2 || true; exit 1; } + echo "Nested target provisioning complete." >&2 + NESTED_SETUP + if [ "${setup_rc}" -ne 0 ]; then + echo "Nested target provisioning failed (rc=${setup_rc})" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -n 200 >&2 || true + exit 1 + fi + for _ in $(seq 1 60); do + podman exec "${TARGET_NAME}" pgrep -u 1000 -f gnome-session >/dev/null && break + sleep 2 + done + podman exec "${TARGET_NAME}" pgrep -u 1000 -f gnome-session >/dev/null || { + echo "GDM did not start a GNOME session for bluefin-test" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -u gdm >&2 || true + podman exec "${TARGET_NAME}" journalctl --no-pager _COMM=gnome-shell >&2 || true + exit 1 + } + + echo "=== nested target environment snapshot ===" >&2 + podman exec "${TARGET_NAME}" bash -c 'tr "\0" " " &2 || true + podman exec "${TARGET_NAME}" bash -c 'ls /run/user/1000; ls -l /dev/uinput; ls /usr/libexec/gnome-ponytail-daemon; python3 -c "import ponytail.ponytail; print(\"ponytail module ok\")"; cat /usr/lib/os-release | head -3' >&2 || true + echo "=== end environment snapshot ===" >&2 + + # behave tags reach the in-session script through a file: + # qecore-headless replaces its environment with the session's. + printf '%s' "${BEHAVE_TAGS}" >/workspace/lane/behave-tags + printf '%s' "${GOOSE_CHECK}" >/workspace/lane/goose-check + cat >/workspace/lane/run-chairlift.sh <<'RUNNER' + #!/bin/bash + # Runs inside the bluefin-test GNOME Wayland session (qecore-headless). + set -uo pipefail + OUT=/tmp/results/atspi + LANE=/workspace/lane + echo "session: XDG_SESSION_TYPE=${XDG_SESSION_TYPE:-} WAYLAND_DISPLAY=${WAYLAND_DISPLAY:-} XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-} DISPLAY=${DISPLAY:-}" + + # Shell ready = Shell.Eval answers (needs --unsafe-mode) with a panel. + shell_ready=false + for attempt in $(seq 1 60); do + if gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.panel ? true : false' 2>/dev/null | grep -q "(true, 'true')"; then + shell_ready=true + break + fi + sleep 2 + done + if [ "${shell_ready}" != true ]; then + echo "GNOME Shell never answered Shell.Eval" >&2 + echo 3 >"${OUT}/behave-rc.txt" + exit 3 + fi + pgrep -u "$(id -u)" -f gnome-ponytail-daemon >/dev/null || { /usr/libexec/gnome-ponytail-daemon & sleep 2; } + + # Wayland only: the window must reach the session's compositor, and + # there is no X server for it to fall back to. + [ -n "${WAYLAND_DISPLAY:-}" ] || { echo "session has no WAYLAND_DISPLAY" >&2; echo 4 >"${OUT}/behave-rc.txt"; exit 4; } + export XDG_SESSION_TYPE=wayland GDK_BACKEND=wayland + unset DISPLAY XAUTHORITY + # The same isolation run_atspi.sh applies, minus the display: memory + # GSettings, no portals, the accessibility bridge on. + export LANG=C LC_ALL=C GSETTINGS_BACKEND=memory GDK_DEBUG=no-portals GTK_A11Y=atspi + unset NO_AT_BRIDGE + export GSETTINGS_SCHEMA_DIR=/home/bluefin-test/chairlift-schemas + export CHAIRLIFT_REQUIRE_ATSPI=1 + export CHAIRLIFT_ATSPI_APP=/workspace/chairlift/build/e2e/chairlift + export CHAIRLIFT_ATSPI_OUT="${OUT}" + CHAIRLIFT_NAVIGATION="$(cat "${LANE}/inventory/navigation.json")" + CHAIRLIFT_SHORTCUTS="$(cat "${LANE}/inventory/shortcuts.json")" + CHAIRLIFT_APP_NAME="$(cat "${LANE}/inventory/app-name.txt")" + export CHAIRLIFT_NAVIGATION CHAIRLIFT_SHORTCUTS CHAIRLIFT_APP_NAME + PY=/home/bluefin-test/atspi-venv/bin/python + + # CHAIRLIFT_REQUIRE_ATSPI: a missing stack fails, it never skips. + if ! "${PY}" -c 'from dogtail.config import config; import behave, pyatspi, gi; gi.require_version("Gtk", "3.0"); gi.require_version("Gdk", "3.0"); import ponytail.ponytail'; then + echo "accessibility stack is unavailable in the Wayland session" >&2 + echo 5 >"${OUT}/behave-rc.txt" + exit 5 + fi + + TAG_ARGS=() + TAGS="$(cat "${LANE}/behave-tags")" + [ -n "${TAGS}" ] && TAG_ARGS=(--tags "${TAGS}") + cd /tmp/results + BEHAVE_RC=0 + timeout 2700 "${PY}" -m behave /workspace/chairlift/test/e2e/features \ + --no-capture --no-capture-stderr \ + --format pretty --outfile "${OUT}/behave.log" \ + --format json.pretty --outfile "${OUT}/results.json" \ + --format progress \ + --junit --junit-directory "${OUT}/junit" \ + "${TAG_ARGS[@]}" || BEHAVE_RC=$? + echo "${BEHAVE_RC}" >"${OUT}/behave-rc.txt" + + # ---- Goose desktop profile isolation (x86_64 only) ---------------- + G=/tmp/results/goose + if [ "$(cat "${LANE}/goose-check")" != "true" ] || [ "$(uname -m)" != "x86_64" ]; then + echo skipped >"${G}/rc.txt" + exit 0 + fi + BREW="$(cat "${G}/brew-path.txt")" + export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_ENV_HINTS=1 HOMEBREW_NO_INSTALL_CLEANUP=1 + GRC=0 + { + echo "+ brew tap ublue-os/tap" && "${BREW}" tap ublue-os/tap && + echo "+ brew install ublue-os/tap/linux-mcp-server" && "${BREW}" install ublue-os/tap/linux-mcp-server && + echo "+ brew install cpio" && "${BREW}" install cpio && + echo "+ brew install --cask ublue-os/tap/goose-linux" && "${BREW}" install --cask ublue-os/tap/goose-linux + } >"${G}/brew-install.log" 2>&1 || GRC=$? + echo "brew install exit ${GRC}" >>"${G}/brew-install.log" + if [ "${GRC}" -ne 0 ]; then + echo "brew-install-failed" >"${G}/rc.txt" + exit 0 + fi + BIN="$(dirname "${BREW}")" + SERVER="${BIN}/linux-mcp-server" + DESKTOP="${BIN}/goose-desktop" + DATA=/home/bluefin-test/goose-isolation + PROFILE="${DATA}/chairlift/troubleshooting" + ls -la "${HOME}/.config" >"${G}/home-config-before.txt" 2>&1 + "${LANE}/bin/gooseprofile" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } + cp "${PROFILE}/goose/config/config.yaml" "${G}/config.before.yaml" + + ( + export GOOSE_PATH_ROOT="${PROFILE}/goose" XDG_CONFIG_HOME="${PROFILE}/desktop" + export GOOSE_PROVIDER=openai OPENAI_HOST=http://127.0.0.1:9 OPENAI_API_KEY=x GOOSE_MODEL=bluefin-active + export ELECTRON_OZONE_PLATFORM_HINT=wayland + exec "${DESKTOP}" --ozone-platform=wayland + ) >"${G}/goose-desktop.log" 2>&1 & + GOOSE_PID=$! + sleep 35 + # Evidence while it runs: SingletonLock is removed on clean exit. + { + echo "alive=$(kill -0 "${GOOSE_PID}" 2>/dev/null && echo yes || echo no)" + echo "profile_lock=$(readlink "${PROFILE}/desktop/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + } >"${G}/lock.txt" + ps -eo pid,args | grep -i -E 'goose|electron' | grep -v grep >"${G}/processes.txt" || true + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/goose-window.png" >"${G}/screenshot.txt" 2>&1 || true + sleep 5 + pkill -TERM -P "${GOOSE_PID}" 2>/dev/null; kill -TERM "${GOOSE_PID}" 2>/dev/null + pkill -TERM -u "$(id -u)" -f 'goose-desktop|goosed|Goose' 2>/dev/null + sleep 5 + pkill -KILL -u "$(id -u)" -f 'goose-desktop|goosed|Goose' 2>/dev/null + wait "${GOOSE_PID}" 2>/dev/null + + cp "${PROFILE}/goose/config/config.yaml" "${G}/config.after.yaml" 2>/dev/null + "${LANE}/bin/gooseprofile" enabled "${PROFILE}/goose/config/config.yaml" >"${G}/enabled-extensions.txt" 2>&1 + find "${PROFILE}" -maxdepth 4 \( -name 'Cache*' -o -name 'GPUCache' \) -prune -o -print | sort >"${G}/profile-tree.txt" + for d in "${HOME}/.config/goose" "${HOME}/.config/Goose" "${HOME}/.local/share/goose" "${HOME}/.local/state/goose"; do + if [ -e "${d}" ]; then echo "PRESENT ${d}"; find "${d}" -maxdepth 3 | head -50; else echo "ABSENT ${d}"; fi + done >"${G}/home-goose-paths.txt" + echo done >"${G}/rc.txt" + exit 0 + RUNNER + chmod 0755 /workspace/lane/run-chairlift.sh + + echo "Running ChairLift's behave suite in the nested Wayland session..." >&2 + QECORE_RC=0 + # The user's name, not 1000:1000, so supplementary groups survive. + podman exec \ + --user bluefin-test \ + --env USER=bluefin-test \ + --env HOME=/home/bluefin-test \ + --env XDG_RUNTIME_DIR=/run/user/1000 \ + --env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + --env AT_SPI_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + --env XDG_SESSION_TYPE=wayland \ + --env XDG_SESSION_DESKTOP=gnome \ + --env LIBGL_ALWAYS_SOFTWARE=1 \ + --env GALLIUM_DRIVER=llvmpipe \ + "${TARGET_NAME}" \ + qecore-headless \ + --session-type wayland \ + --session-desktop gnome \ + /workspace/lane/run-chairlift.sh || QECORE_RC=$? + echo "qecore-headless exit ${QECORE_RC}" >&2 + + # ---- verdicts ------------------------------------------------------ + BEHAVE_RC=$(cat /tmp/results/atspi/behave-rc.txt 2>/dev/null || printf '%s' "${QECORE_RC}") + SUITE_RC=0 + python3 - <<'PY' || SUITE_RC=$? + import json, os, re, sys + out = "/tmp/results" + path = os.path.join(out, "atspi", "results.json") + if not os.path.exists(path): + open(os.path.join(out, "result-summary.txt"), "w").write("Execution failed: no behave results") + sys.exit(1) + data = json.load(open(path, encoding="utf-8")) + counts = {"passed": 0, "failed": 0, "skipped": 0, "other": 0} + failed = [] + per_feature = {} + safe = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .,:;()/_+'\"—-]{0,159}$") + for feature in data: + fc = per_feature.setdefault(feature.get("name", "?"), {"passed": 0, "failed": 0, "skipped": 0, "other": 0}) + for scenario in feature.get("elements", []): + if scenario.get("type") == "background": + continue + status = scenario.get("status", "other") + key = status if status in counts else "other" + counts[key] += 1 + fc[key] += 1 + if status == "failed" and safe.fullmatch(scenario.get("name", "")): + failed.append(scenario["name"]) + executed = counts["passed"] + counts["failed"] + counts["other"] + summary = f"{counts['passed']}/{executed} executed scenarios passed ({counts['failed']} failed, {counts['skipped']} skipped, {counts['other']} other)" + open(os.path.join(out, "result-summary.txt"), "w").write(summary) + json.dump(failed[:20], open(os.path.join(out, "failed-scenarios.json"), "w"), separators=(",", ":")) + json.dump({"counts": counts, "features": per_feature, "failed": failed}, + open(os.path.join(out, "scenario-counts.json"), "w"), indent=1) + print(summary) + for name, fc in per_feature.items(): + print(f" {name}: {fc}") + if executed == 0: + print("ERROR: 0 scenarios executed; a run with no executed tests cannot pass", file=sys.stderr) + sys.exit(1) + PY + + # Failed scenarios: the accessibility tree and the log tail, since + # Argo logs are the only view of a container lane. + for dir in /tmp/results/atspi/scenarios/*/; do + [ -f "${dir}tree.txt" ] || continue + echo "=== failed scenario: $(basename "${dir}") ===" >&2 + ls -l "${dir}" >&2 + head -n 120 "${dir}tree.txt" >&2 + tail -n 40 "${dir}chairlift.log" >&2 2>/dev/null || true + done + + GOOSE_RC=0 + G=/tmp/results/goose + GOOSE_STATE=$(cat "${G}/rc.txt" 2>/dev/null || echo not-run) + if [[ "${GOOSE_STATE}" == "skipped" ]]; then + echo '{"status":"skipped"}' >"${G}/verdict.json" + else + echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 + cat "${G}/brew-install.log" >&2 2>/dev/null || true + GOOSE_STATE="${GOOSE_STATE}" python3 - <<'PY' || GOOSE_RC=$? + import json, os, sys + g = "/tmp/results/goose" + def read(name): + try: + return open(os.path.join(g, name), encoding="utf-8", errors="replace").read() + except OSError: + return "" + state = os.environ["GOOSE_STATE"] + lock = dict(line.split("=", 1) for line in read("lock.txt").split() if "=" in line) + enabled = sorted(read("enabled-extensions.txt").split()) + paths = read("home-goose-paths.txt") + tree = read("profile-tree.txt").splitlines() + root = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting/goose" + checks = { + "brew_install_succeeded": state == "done", + "goose_was_running_at_35s": lock.get("alive") == "yes", + "singleton_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", + "no_singleton_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", + "only_linux_tools_and_bluefin_knowledge_enabled": enabled == ["bluefin-knowledge", "linux-tools"], + "goose_data_or_state_under_path_root": any(p.startswith(root + "/data/") or p.startswith(root + "/state/") for p in tree), + "nothing_in_home_config_goose": "ABSENT /home/bluefin-test/.config/goose" in paths, + } + verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, + "checks": checks, "lock": lock, "enabled": enabled} + json.dump(verdict, open(os.path.join(g, "verdict.json"), "w"), indent=1) + print(json.dumps(verdict, indent=1)) + print(paths) + sys.exit(0 if verdict["status"] == "pass" else 1) + PY + tail -n 60 "${G}/goose-desktop.log" >&2 2>/dev/null || true + fi + + echo "behave exit ${BEHAVE_RC}; suite verdict ${SUITE_RC}; goose verdict ${GOOSE_RC}" >&2 + if [[ "${BEHAVE_RC}" != "0" || "${SUITE_RC}" != "0" || "${GOOSE_RC}" != "0" ]]; then + exit 1 + fi + volumes: + - name: workspace + emptyDir: {} + - name: results + emptyDir: {} + - name: containers-storage + emptyDir: {} + - name: containers-runroot + emptyDir: {} + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: results-store + hostPath: + path: /var/mnt/ghost-data/local-path + type: Directory + - name: pip-cache + hostPath: + path: /var/cache/bluefin-qa-pip + type: DirectoryOrCreate diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py new file mode 100644 index 0000000000..e5695acfe1 --- /dev/null +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -0,0 +1,141 @@ +"""Static contract for run-chairlift-wayland-tests (ChairLift's live Wayland lane).""" + +import re +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATE = ROOT / "argo/workflow-templates/run-chairlift-wayland-tests.yaml" + + +def _doc(): + return yaml.safe_load(TEMPLATE.read_text(encoding="utf-8")) + + +def _template(): + return next(t for t in _doc()["spec"]["templates"] if t["name"] == "run-chairlift-wayland-tests") + + +def _runner(): + return _template()["script"]["source"] + + +def _heredoc(source, tag): + match = re.search(rf"<<'{tag}'[^\n]*\n(.*?)\n\s*{tag}\n", source, re.S) + assert match, f"heredoc {tag} not found" + return match.group(1) + + +def _bash_parses(script): + result = subprocess.run(["bash", "-n"], input=script, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + + +def test_parameters_are_the_lane_contract(): + names = [p["name"] for p in _doc()["spec"]["arguments"]["parameters"]] + assert names == [ + "chairlift-repo", + "chairlift-branch", + "image", + "image-tag", + "behave-tags", + "goose-isolation-check", + ] + defaults = {p["name"]: p["value"] for p in _doc()["spec"]["arguments"]["parameters"]} + assert defaults["image"] == "ghcr.io/projectbluefin/dakota" + assert defaults["image-tag"] == "testing" + + +def test_runs_on_ghost_behind_the_container_qa_semaphore(): + template = _template() + assert template["nodeSelector"]["kubernetes.io/hostname"] == "ghost" + keys = [s["configMapKeyRef"]["key"] for s in template["synchronization"]["semaphores"]] + assert keys == ["ghost-container-qa"] + assert template["activeDeadlineSeconds"] <= 5400 + + +def test_builds_the_requested_branch_with_the_e2e_target(): + build = _template()["initContainers"][0] + script = build["args"][0] + assert 'git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}"' in script + assert "CGO_ENABLED=0 make build-e2e schemas" in script + assert "troubleshoot.ProfileAt(os.Args[2])" in script + assert "navigation.Items()" in script + _bash_parses(script) + + +def test_boots_the_run_container_tests_wayland_target(): + source = _runner() + setup = _heredoc(source, "NESTED_SETUP") + assert "--systemd=always" in source and "--privileged" in source + # Rule 12: headless shell, never DRM master, Shell.Eval kept. + assert "--mode=%i --unsafe-mode --headless --virtual-monitor 1920x1080" in setup + assert "loginctl enable-linger bluefin-test" in setup + assert 'chage -d "$(date +%Y-%m-%d)" bluefin-test' in setup + assert "--volume /etc/resolv.conf:/etc/resolv.conf:ro" in source + assert "qecore-headless \\\n --session-type wayland" in source + assert "systemctl --global mask brew-preinstall.service" in setup + _bash_parses(source) + _bash_parses(setup) + + +def test_wayland_only_never_x11(): + text = TEMPLATE.read_text(encoding="utf-8") + for banned in ("Xvfb", "xvfb", "GDK_BACKEND=x11", "xwd", "DISPLAY=:"): + assert banned not in text, banned + runner = _heredoc(_runner(), "RUNNER") + assert "GDK_BACKEND=wayland" in runner + assert "unset DISPLAY XAUTHORITY" in runner + assert "ELECTRON_OZONE_PLATFORM_HINT=wayland" in runner + assert "org.gnome.Shell.Screenshot" in runner + _bash_parses(runner) + + +def test_suite_isolation_matches_the_fixture_harness(): + source = _runner() + runner = _heredoc(source, "RUNNER") + # A shipped config.yml outranks every fixture; the suite refuses it. + assert "--tmpfs /usr/share/chairlift:ro,notmpcopyup" in source + assert "destination=/proc/cmdline,ro" in source + for setting in ("GSETTINGS_BACKEND=memory", "GDK_DEBUG=no-portals", "GTK_A11Y=atspi", "CHAIRLIFT_REQUIRE_ATSPI=1"): + assert setting in runner, setting + assert "--require-hashes -r /workspace/chairlift/test/e2e/requirements-atspi.txt" in source + + +def test_fails_closed_and_cleans_up(): + source = _runner() + assert "trap on_exit EXIT" in source + assert "trap 'exit 143' TERM" in source + assert "trap 'exit 130' INT" in source + assert 'podman rm --force "${TARGET_NAME}"' in source + assert "0 scenarios executed" in source + assert "sys.exit(1)" in source + + +def test_goose_check_asserts_profile_isolation(): + runner = _heredoc(_runner(), "RUNNER") + for command in ( + '"${BREW}" install ublue-os/tap/linux-mcp-server', + '"${BREW}" install cpio', + '"${BREW}" install --cask ublue-os/tap/goose-linux', + ): + assert command in runner, command + for setting in ( + 'GOOSE_PATH_ROOT="${PROFILE}/goose"', + 'XDG_CONFIG_HOME="${PROFILE}/desktop"', + "GOOSE_PROVIDER=openai", + "OPENAI_HOST=http://127.0.0.1:9", + "GOOSE_MODEL=bluefin-active", + ): + assert setting in runner, setting + source = _runner() + for check in ( + "singleton_lock_in_profile", + "no_singleton_lock_in_home_config", + "only_linux_tools_and_bluefin_knowledge_enabled", + "goose_data_or_state_under_path_root", + "nothing_in_home_config_goose", + ): + assert check in source, check From 9027b45f6c0edca7c166026e161ca0c8c4f45611 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 13:32:22 -0400 Subject: [PATCH 02/19] fix(argo): make the ChairLift Wayland lane run end to end - Read WAYLAND_DISPLAY from the user manager: qecore hands the script gnome-session's environment, which predates the compositor. - Install the Goose packages under brew shellenv, as chairlift-wrapper does; the goose-linux cask's bare rpm2cpio | cpio preflight exits 127 without Homebrew's bin on PATH. - Mask Dakota's brew-preinstall unit and brew timers so they cannot hold Homebrew's locks; python-uinput is best-effort (input goes through gnome-ponytail-daemon). - Document the lane and its Wayland traps. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 31 +++++++++++++--- docs/reference/workflow-reference.md | 37 +++++++++++++++++++ docs/skills/argo-workflows/patterns.md | 22 +++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index baec6916e0..e07793c083 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -9,8 +9,8 @@ metadata: (test/e2e/features) inside a live GNOME Wayland session: the nested, systemd-booted Dakota target run-container-tests boots (headless GNOME Shell with a virtual monitor, GDM autologin, test-user linger), driven by - qecore-headless --session-type wayland. No X server is started; the - application runs with GDK_BACKEND=wayland and DISPLAY unset. On x86_64 it + qecore-headless --session-type wayland. The application (and Goose's + Electron window) run with GDK_BACKEND=wayland and DISPLAY unset. On x86_64 it then installs Enhanced Troubleshooting's packages through the target's Homebrew and checks that Goose desktop launched in ChairLift's profile writes nothing outside that profile. @@ -622,9 +622,25 @@ spec: fi pgrep -u "$(id -u)" -f gnome-ponytail-daemon >/dev/null || { /usr/libexec/gnome-ponytail-daemon & sleep 2; } - # Wayland only: the window must reach the session's compositor, and - # there is no X server for it to fall back to. - [ -n "${WAYLAND_DISPLAY:-}" ] || { echo "session has no WAYLAND_DISPLAY" >&2; echo 4 >"${OUT}/behave-rc.txt"; exit 4; } + # Wayland only: the window must reach the session's compositor + # natively, never through the session's on-demand Xwayland. + # qecore-headless hands the script gnome-session's environment, which + # predates the compositor; gnome-shell exports WAYLAND_DISPLAY to the + # user manager once its socket is up, so read it from there. + if [ -z "${WAYLAND_DISPLAY:-}" ]; then + WAYLAND_DISPLAY=$(systemctl --user show-environment 2>/dev/null | sed -n 's/^WAYLAND_DISPLAY=//p') + fi + case "${WAYLAND_DISPLAY:-}" in + /*) WAYLAND_SOCKET="${WAYLAND_DISPLAY}" ;; + *) WAYLAND_SOCKET="${XDG_RUNTIME_DIR}/${WAYLAND_DISPLAY:-}" ;; + esac + if [ -z "${WAYLAND_DISPLAY:-}" ] || [ ! -S "${WAYLAND_SOCKET}" ]; then + echo "the session exposes no Wayland socket (WAYLAND_DISPLAY='${WAYLAND_DISPLAY:-}')" >&2 + echo 4 >"${OUT}/behave-rc.txt" + exit 4 + fi + export WAYLAND_DISPLAY + echo "Wayland socket: ${WAYLAND_SOCKET}" export XDG_SESSION_TYPE=wayland GDK_BACKEND=wayland unset DISPLAY XAUTHORITY # The same isolation run_atspi.sh applies, minus the display: memory @@ -670,6 +686,11 @@ spec: fi BREW="$(cat "${G}/brew-path.txt")" export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_ENV_HINTS=1 HOMEBREW_NO_INSTALL_CLEANUP=1 + # The environment ChairLift's desktop entry gives brew: chairlift-wrapper + # evaluates `brew shellenv`. The goose-linux cask's preflight runs a + # bare `rpm2cpio | cpio`, which finds neither without Homebrew's bin + # on PATH (observed: exit 127 from the session's own PATH). + eval "$("${BREW}" shellenv)" GRC=0 { echo "+ brew tap ublue-os/tap" && "${BREW}" tap ublue-os/tap && diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index c6c0a0d1ad..33beea9ff6 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -14,6 +14,7 @@ template add or rename. - [dakota-build-pipeline](#dakota-build-pipeline) - [zot-candidate-lifecycle](#zot-candidate-lifecycle) - [bluefin-server-build-pipeline](#bluefin-server-build-pipeline) + - [run-chairlift-wayland-tests](#run-chairlift-wayland-tests) - [KubeStellar Workflows](#kubestellar-workflows) - [Supporting Templates](#supporting-templates) - [Distributed Build/RE Grid](#distributed-buildre-grid) @@ -126,6 +127,40 @@ before retrying. and execution are identical to dakota. - **Cache policy:** CAS (`cache.storage-service`) and action cache use the shared Buildbarn frontend (`frontend.buildbarn.svc.cluster.local:8980`); artifacts and sources are indexed in `bb-remote-asset` (`:8984`) and pushed there, with upstream caches as read-only fallbacks. Source caches override the project entries. See `manifests/buildstream-remote-cache-config.yaml`. +### run-chairlift-wayland-tests +- **Purpose:** ChairLift's behave + dogtail AT-SPI suite + (`test/e2e/features`) in a live GNOME **Wayland** session, plus an + Enhanced Troubleshooting Goose-profile isolation check on x86_64. +- **Parameters:** `chairlift-repo`, `chairlift-branch`, `image` + (`ghcr.io/projectbluefin/dakota`), `image-tag` (`testing`), `behave-tags` + (empty = every scenario), `goose-isolation-check` (`true`). +- **Shape:** a `golang:1.26` init container clones the branch and runs + `make build-e2e schemas`, plus two lane tools built inside the module: + `navjson` (the page/shortcut inventory the Go gate normally passes) and + `gooseprofile` (`troubleshoot.ProfileAt(dir).Write`, and a config reader). + The runner then boots `run-container-tests`' nested systemd/GDM target — + headless GNOME Shell drop-in, linger, `lastchg`, resolver bind — with + `/usr/share/chairlift` masked and `/proc/cmdline` blanked like + `dakota_atspi.sh`, and runs behave from a hash-pinned venv under + `qecore-headless --session-type wayland`, with `GDK_BACKEND=wayland` and + `DISPLAY` unset, so neither the app nor Goose can fall back to the + session's on-demand Xwayland. Xvfb is never used. +- **Wayland traps:** qecore hands the script gnome-session's environment, + which has no `WAYLAND_DISPLAY`; the script reads it from + `systemctl --user show-environment`. Mutter's focus-stealing prevention + leaves each new window unfocused, so ChairLift's `environment.py` + activates it through `Shell.Eval` (needs `--unsafe-mode`). Dakota's + `brew-preinstall.service` and brew timers are masked so they cannot hold + Homebrew's locks during the Goose check. +- **Verdicts:** fails when behave fails, when 0 scenarios executed, or when + any Goose check fails. Outputs `result`, `failed-scenarios`, and + `goose-isolation` (JSON). Everything under `/tmp/results` (behave log, + JUnit, per-scenario `chairlift.log`, `tree.txt`, `screen.png`; Goose + `brew-install.log`, lock evidence, config, screenshot) is persisted to + ghost's result store under `/chairlift-wayland/`. +- **Submit:** not deployed until merged; until then submit the template + file as a Workflow (`kind: Workflow`, `generateName`) with `argo submit`. + ## KubeStellar Workflows KubeStellar installation and upgrades are owned by the `kubestellar-applications` @@ -248,5 +283,7 @@ Pod resource requests/limits used by workflow steps: | Template / container | CPU req/limit | Memory req/limit | | --- | --- | --- | | `run-container-tests` | 2 / 4 | 4Gi / 8Gi | +| `run-chairlift-wayland-tests` `main` | 2 / 4 | 4Gi / 8Gi | +| `run-chairlift-wayland-tests` `build-chairlift` | 2 / 4 | 2Gi / 4Gi | | `bst-build-re` `main` | 2 / 4 | 4Gi / 8Gi | | `bst-build-re` `publish` | 1 / 16 | 256Mi / 2Gi | diff --git a/docs/skills/argo-workflows/patterns.md b/docs/skills/argo-workflows/patterns.md index 076f517044..79f81b9844 100644 --- a/docs/skills/argo-workflows/patterns.md +++ b/docs/skills/argo-workflows/patterns.md @@ -137,6 +137,28 @@ NVIDIA image builds are disabled for this clean distributed-build path. and `image-poll-dakota` (not suspended) drives it on every new `dakota:testing` digest. +### Driving a GTK app through AT-SPI in the nested Wayland session + +`run-chairlift-wayland-tests` runs an application's own behave suite inside +`run-container-tests`' nested GNOME session. Three traps, each observed on +ghost: + +- **qecore's script environment has no `WAYLAND_DISPLAY`.** qecore-headless + copies gnome-session's environment, which predates the compositor, and sets + `DISPLAY=:0` (Xwayland). Read `WAYLAND_DISPLAY` from + `systemctl --user show-environment`, check the socket, set + `GDK_BACKEND=wayland`, and unset `DISPLAY` so nothing falls back to X. +- **New windows are not focused.** Mutter's focus-stealing prevention leaves a + window mapped without an activation token unfocused (`focus_window` is + null), so every synthesized key reaches nothing. Activate it with + `Main.activateWindow` over `Shell.Eval` (the headless drop-in's + `--unsafe-mode` allows it) and wait for `global.display.focus_window`. +- **The first ponytail key is dropped** while gnome-ponytail-daemon's + remote-desktop session starts. Spend it on a bare Shift once per run. + +Screenshots come from `org.gnome.Shell.Screenshot.Screenshot`, which +`--unsafe-mode` also opens to the test user; there is no root window for `xwd`. + ## Concurrency and scheduling ### VM concurrency: k8s native scheduling, no semaphores From 67e882d656905e160c9baf1048e117c46d633a40 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 13:38:55 -0400 Subject: [PATCH 03/19] feat(argo): install the Goose packages through ChairLift's own Setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The isolation check proved the goose-linux cask only under brew shellenv, not ChairLift's install path. It now runs troubleshoot.Setup(Detect(), …) from a lane tool built against the branch, with PATH=/usr/bin:/bin and no shellenv, the way a direct chairlift launch installs; asserts it exits 0 and goose-desktop resolves; and asserts troubleshoot.Command's PATH starts with Homebrew's bin. The profile isolation assertions are unchanged. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 135 ++++++++++++------ docs/reference/workflow-reference.md | 15 +- .../unit/test_run_chairlift_wayland_tests.py | 20 ++- 3 files changed, 122 insertions(+), 48 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index e07793c083..19414bfe1d 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -11,9 +11,9 @@ metadata: Shell with a virtual monitor, GDM autologin, test-user linger), driven by qecore-headless --session-type wayland. The application (and Goose's Electron window) run with GDK_BACKEND=wayland and DISPLAY unset. On x86_64 it - then installs Enhanced Troubleshooting's packages through the target's - Homebrew and checks that Goose desktop launched in ChairLift's profile - writes nothing outside that profile. + then installs Enhanced Troubleshooting's packages through ChairLift's own + troubleshoot.Setup, with no Homebrew on PATH, and checks that Goose + desktop launched in ChairLift's profile writes nothing outside it. labels: app.kubernetes.io/component: test-runner app.kubernetes.io/part-of: bluefin-test-suite @@ -123,7 +123,7 @@ spec: # navjson: the page and shortcut inventories the Go gate # (test/e2e/atspi_behave_test.go) hands the suite, from the one # authority, internal/navigation. - mkdir -p build/lane/navjson build/lane/gooseprofile + mkdir -p build/lane/navjson build/lane/troubleshootcheck cat >build/lane/navjson/main.go <<'GO' package main @@ -169,35 +169,76 @@ spec: } } GO - # gooseprofile: writes Enhanced Troubleshooting's Goose profile with - # the branch's own internal/troubleshoot, and reads back which - # extensions a config.yaml leaves enabled. - cat >build/lane/gooseprofile/main.go <<'GO' + # troubleshootcheck: drives the branch's own internal/troubleshoot. + # setup troubleshoot.Setup(troubleshoot.Detect(), progress): + # the tap and the three installs, through + # internal/homebrew exactly as the app runs them. + # command-path the PATH troubleshoot.Command hands the launch. + # write troubleshoot.ProfileAt(dir).Write(server). + # enabled the extensions a config.yaml leaves enabled. + cat >build/lane/troubleshootcheck/main.go <<'GO' package main import ( "fmt" "os" "sort" + "strings" "github.com/projectbluefin/chairlift/internal/troubleshoot" "gopkg.in/yaml.v3" ) + func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + func printState(s troubleshoot.State) { + fmt.Printf("supported=%t\nserver=%s\ndesktop=%s\nllmman=%s\n", s.Supported, s.ServerPath, s.DesktopPath, s.LLMManPath) + } + func main() { switch { + case len(os.Args) == 2 && os.Args[1] == "setup": + fmt.Println("PATH=" + os.Getenv("PATH")) + state, err := troubleshoot.Setup(troubleshoot.Detect(), func(step string) { + fmt.Println("progress: " + step) + }) + printState(state) + if err != nil { + fail(err) + } + if !state.Installed() { + fail(fmt.Errorf("setup returned without every package installed")) + } + case len(os.Args) == 3 && os.Args[1] == "command-path": + state := troubleshoot.Detect() + // llmman is Agent Mode's install, not this feature's; any + // absolute executable satisfies Ready for an environment read. + if state.LLMManPath == "" { + state.LLMManPath = "/usr/bin/true" + } + cmd, err := troubleshoot.Command(state, troubleshoot.ProfileAt(os.Args[2]), "bluefin-active") + if err != nil { + fail(err) + } + fmt.Println("argv=" + strings.Join(cmd.Args, " ")) + for _, kv := range cmd.Env { + if strings.HasPrefix(kv, "PATH=") || strings.HasPrefix(kv, "GOOSE_PATH_ROOT=") || strings.HasPrefix(kv, "XDG_CONFIG_HOME=") { + fmt.Println(kv) + } + } case len(os.Args) == 4 && os.Args[1] == "write": profile := troubleshoot.ProfileAt(os.Args[2]) if err := profile.Write(os.Args[3]); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + fail(err) } fmt.Println(profile.ConfigPath()) case len(os.Args) == 3 && os.Args[1] == "enabled": data, err := os.ReadFile(os.Args[2]) if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + fail(err) } var cfg struct { Extensions map[string]struct { @@ -205,8 +246,7 @@ spec: } `yaml:"extensions"` } if err := yaml.Unmarshal(data, &cfg); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) + fail(err) } var names []string for name, ext := range cfg.Extensions { @@ -219,13 +259,13 @@ spec: fmt.Println(name) } default: - fmt.Fprintln(os.Stderr, "usage: gooseprofile write | enabled ") + fmt.Fprintln(os.Stderr, "usage: troubleshootcheck setup | command-path | write | enabled ") os.Exit(2) } } GO CGO_ENABLED=0 go build -o /workspace/lane/bin/navjson ./build/lane/navjson - CGO_ENABLED=0 go build -o /workspace/lane/bin/gooseprofile ./build/lane/gooseprofile + CGO_ENABLED=0 go build -o /workspace/lane/bin/troubleshootcheck ./build/lane/troubleshootcheck /workspace/lane/bin/navjson /workspace/lane/inventory cat /workspace/lane/inventory/navigation.json; echo chmod -R a+rX /workspace @@ -685,31 +725,33 @@ spec: exit 0 fi BREW="$(cat "${G}/brew-path.txt")" - export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_ENV_HINTS=1 HOMEBREW_NO_INSTALL_CLEANUP=1 - # The environment ChairLift's desktop entry gives brew: chairlift-wrapper - # evaluates `brew shellenv`. The goose-linux cask's preflight runs a - # bare `rpm2cpio | cpio`, which finds neither without Homebrew's bin - # on PATH (observed: exit 127 from the session's own PATH). - eval "$("${BREW}" shellenv)" - GRC=0 + DATA=/home/bluefin-test/goose-isolation + PROFILE="${DATA}/chairlift/troubleshooting" + # The install goes through ChairLift itself — troubleshoot.Setup over + # internal/homebrew — with a PATH that deliberately lacks Homebrew's + # bin and no `brew shellenv`, which is how a direct `chairlift` launch + # runs it. ChairLift has to find brew through its fallback and hand + # brew's children Homebrew's bin itself: the goose-linux cask's + # preflight runs a bare `rpm2cpio | cpio` (exit 127 without it). + # Auto-update is off only to bound the lane's time. + BARE_ENV=(env -u HOMEBREW_PREFIX -u HOMEBREW_CELLAR -u HOMEBREW_REPOSITORY + PATH=/usr/bin:/bin HOMEBREW_NO_AUTO_UPDATE=1) + SRC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 || SRC=$? + echo "${SRC}" >"${G}/setup-rc.txt" + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 || true { - echo "+ brew tap ublue-os/tap" && "${BREW}" tap ublue-os/tap && - echo "+ brew install ublue-os/tap/linux-mcp-server" && "${BREW}" install ublue-os/tap/linux-mcp-server && - echo "+ brew install cpio" && "${BREW}" install cpio && - echo "+ brew install --cask ublue-os/tap/goose-linux" && "${BREW}" install --cask ublue-os/tap/goose-linux - } >"${G}/brew-install.log" 2>&1 || GRC=$? - echo "brew install exit ${GRC}" >>"${G}/brew-install.log" - if [ "${GRC}" -ne 0 ]; then - echo "brew-install-failed" >"${G}/rc.txt" + "${BREW}" list --versions linux-mcp-server cpio rpm2cpio + "${BREW}" list --cask --versions goose-linux + } >"${G}/brew-installed.txt" 2>&1 || true + if [ "${SRC}" -ne 0 ]; then + echo "setup-failed" >"${G}/rc.txt" exit 0 fi - BIN="$(dirname "${BREW}")" - SERVER="${BIN}/linux-mcp-server" - DESKTOP="${BIN}/goose-desktop" - DATA=/home/bluefin-test/goose-isolation - PROFILE="${DATA}/chairlift/troubleshooting" + SERVER="$(sed -n 's/^server=//p' "${G}/setup.log")" + DESKTOP="$(sed -n 's/^desktop=//p' "${G}/setup.log")" ls -la "${HOME}/.config" >"${G}/home-config-before.txt" 2>&1 - "${LANE}/bin/gooseprofile" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } + "${LANE}/bin/troubleshootcheck" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } cp "${PROFILE}/goose/config/config.yaml" "${G}/config.before.yaml" ( @@ -737,7 +779,7 @@ spec: wait "${GOOSE_PID}" 2>/dev/null cp "${PROFILE}/goose/config/config.yaml" "${G}/config.after.yaml" 2>/dev/null - "${LANE}/bin/gooseprofile" enabled "${PROFILE}/goose/config/config.yaml" >"${G}/enabled-extensions.txt" 2>&1 + "${LANE}/bin/troubleshootcheck" enabled "${PROFILE}/goose/config/config.yaml" >"${G}/enabled-extensions.txt" 2>&1 find "${PROFILE}" -maxdepth 4 \( -name 'Cache*' -o -name 'GPUCache' \) -prune -o -print | sort >"${G}/profile-tree.txt" for d in "${HOME}/.config/goose" "${HOME}/.config/Goose" "${HOME}/.local/share/goose" "${HOME}/.local/state/goose"; do if [ -e "${d}" ]; then echo "PRESENT ${d}"; find "${d}" -maxdepth 3 | head -50; else echo "ABSENT ${d}"; fi @@ -825,7 +867,10 @@ spec: echo '{"status":"skipped"}' >"${G}/verdict.json" else echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - cat "${G}/brew-install.log" >&2 2>/dev/null || true + for f in setup.log setup-rc.txt command-env.txt brew-installed.txt; do + echo "--- ${f}" >&2 + cat "${G}/${f}" >&2 2>/dev/null || true + done GOOSE_STATE="${GOOSE_STATE}" python3 - <<'PY' || GOOSE_RC=$? import json, os, sys g = "/tmp/results/goose" @@ -840,8 +885,18 @@ spec: paths = read("home-goose-paths.txt") tree = read("profile-tree.txt").splitlines() root = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting/goose" + def kv(name): + return dict(line.split("=", 1) for line in read(name).splitlines() if "=" in line) + setup = kv("setup.log") + command_path = kv("command-env.txt").get("PATH", "") + brew_bin = os.path.dirname(read("brew-path.txt").strip()) checks = { - "brew_install_succeeded": state == "done", + "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", + "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", + "isolation_run_completed": state == "done", + "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), + "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin + and command_path.split(":")[1:] == ["/usr/bin", "/bin"], "goose_was_running_at_35s": lock.get("alive") == "yes", "singleton_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", "no_singleton_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index 33beea9ff6..f5ea389782 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -137,7 +137,9 @@ before retrying. - **Shape:** a `golang:1.26` init container clones the branch and runs `make build-e2e schemas`, plus two lane tools built inside the module: `navjson` (the page/shortcut inventory the Go gate normally passes) and - `gooseprofile` (`troubleshoot.ProfileAt(dir).Write`, and a config reader). + `troubleshootcheck` (drives the branch's `internal/troubleshoot`: + `Setup(Detect(), …)`, the PATH `Command` hands the launch, + `ProfileAt(dir).Write`, and a config reader). The runner then boots `run-container-tests`' nested systemd/GDM target — headless GNOME Shell drop-in, linger, `lastchg`, resolver bind — with `/usr/share/chairlift` masked and `/proc/cmdline` blanked like @@ -152,11 +154,20 @@ before retrying. activates it through `Shell.Eval` (needs `--unsafe-mode`). Dakota's `brew-preinstall.service` and brew timers are masked so they cannot hold Homebrew's locks during the Goose check. +- **Goose check:** ChairLift installs the packages itself + (`troubleshoot.Setup`: tap, `linux-mcp-server`, `cpio`, the `goose-linux` + cask) with `PATH=/usr/bin:/bin` and no `brew shellenv`, as a direct + `chairlift` launch would; the cask's bare `rpm2cpio | cpio` preflight + exits 127 unless ChairLift puts Homebrew's bin on brew's PATH. The check + then asserts `goose-desktop` resolves, `Command`'s PATH starts with + Homebrew's bin, and Goose launched in the profile keeps its lock, data, + state, and config there with only `linux-tools` and `bluefin-knowledge` + enabled. - **Verdicts:** fails when behave fails, when 0 scenarios executed, or when any Goose check fails. Outputs `result`, `failed-scenarios`, and `goose-isolation` (JSON). Everything under `/tmp/results` (behave log, JUnit, per-scenario `chairlift.log`, `tree.txt`, `screen.png`; Goose - `brew-install.log`, lock evidence, config, screenshot) is persisted to + `setup.log`, `command-env.txt`, lock evidence, config, screenshot) is persisted to ghost's result store under `/chairlift-wayland/`. - **Submit:** not deployed until merged; until then submit the template file as a Workflow (`kind: Workflow`, `generateName`) with `argo submit`. diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index e5695acfe1..2492a433b8 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -62,6 +62,9 @@ def test_builds_the_requested_branch_with_the_e2e_target(): assert 'git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}"' in script assert "CGO_ENABLED=0 make build-e2e schemas" in script assert "troubleshoot.ProfileAt(os.Args[2])" in script + # The install goes through ChairLift's own Setup, not hand-written brew. + assert "troubleshoot.Setup(troubleshoot.Detect()," in script + assert "troubleshoot.Command(state," in script assert "navigation.Items()" in script _bash_parses(script) @@ -116,12 +119,13 @@ def test_fails_closed_and_cleans_up(): def test_goose_check_asserts_profile_isolation(): runner = _heredoc(_runner(), "RUNNER") - for command in ( - '"${BREW}" install ublue-os/tap/linux-mcp-server', - '"${BREW}" install cpio', - '"${BREW}" install --cask ublue-os/tap/goose-linux', - ): - assert command in runner, command + # ChairLift installs the packages itself, with no Homebrew on PATH and + # no `brew shellenv`, the way a direct launch runs it. + assert "PATH=/usr/bin:/bin" in runner + assert '"${LANE}/bin/troubleshootcheck" setup' in runner + assert '"${LANE}/bin/troubleshootcheck" command-path' in runner + assert 'eval "$("${BREW}" shellenv)"' not in runner + assert '"${BREW}" install' not in runner for setting in ( 'GOOSE_PATH_ROOT="${PROFILE}/goose"', 'XDG_CONFIG_HOME="${PROFILE}/desktop"', @@ -132,6 +136,10 @@ def test_goose_check_asserts_profile_isolation(): assert setting in runner, setting source = _runner() for check in ( + "setup_ran_without_brew_on_path", + "chairlift_setup_exit_zero", + "goose_desktop_resolves_after_setup", + "command_env_path_has_brew_bin_first", "singleton_lock_in_profile", "no_singleton_lock_in_home_config", "only_linux_tools_and_bluefin_knowledge_enabled", From 00d4330de45f7f0da459cf15fbe2c0db1f5be0dd Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 14:02:27 -0400 Subject: [PATCH 04/19] feat(argo): verify the Enhanced Troubleshooting integration end to end The Goose check no longer satisfies troubleshoot.Command with a stand-in llmman or launches Goose directly. Through ChairLift's own code it now enables Agent Mode (aistack.Enable, WaitHealthy), resolves and pulls the smallest Qwen3 through the alias path (PullModel, ConfigureActiveModel), launches the real troubleshoot.Command in the Wayland session and reads the Goose process's environment, then asks goose (llmman launch goose, same profile) a question that needs linux-tools and search_knowledge and extracts the tool calls from the session database. A model that will not call tools is retried once a size up and otherwise fails the check. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 239 ++++++++++++++++-- .../unit/test_run_chairlift_wayland_tests.py | 34 ++- 2 files changed, 236 insertions(+), 37 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 19414bfe1d..9e3c5ffcac 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -173,18 +173,28 @@ spec: # setup troubleshoot.Setup(troubleshoot.Detect(), progress): # the tap and the three installs, through # internal/homebrew exactly as the app runs them. + # agent-mode aistack.Enable, then aistack.WaitHealthy. + # model resolves through aistack's live resolver, + # ranks it, then aistack.PullModel and + # aistack.ConfigureActiveModel (the alias path). # command-path the PATH troubleshoot.Command hands the launch. + # launch writes the profile and runs the real + # troubleshoot.Command(Detect(), profile, alias). # write troubleshoot.ProfileAt(dir).Write(server). # enabled the extensions a config.yaml leaves enabled. cat >build/lane/troubleshootcheck/main.go <<'GO' package main import ( + "context" + "errors" "fmt" "os" "sort" "strings" + "time" + "github.com/projectbluefin/chairlift/internal/aistack" "github.com/projectbluefin/chairlift/internal/troubleshoot" "gopkg.in/yaml.v3" ) @@ -212,14 +222,78 @@ spec: if !state.Installed() { fail(fmt.Errorf("setup returned without every package installed")) } - case len(os.Args) == 3 && os.Args[1] == "command-path": + case len(os.Args) == 2 && os.Args[1] == "agent-mode": + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute) + defer cancel() + fmt.Println("PATH=" + os.Getenv("PATH")) + start := time.Now() + if err := aistack.Enable(ctx); err != nil { + fmt.Printf("enable_seconds=%.0f\n", time.Since(start).Seconds()) + fail(err) + } + fmt.Printf("enable_seconds=%.0f\nllmman=%s\n", time.Since(start).Seconds(), aistack.Executable()) + if !aistack.WaitHealthy(ctx, 3*time.Minute) { + fail(errors.New("llmman never answered /llmman/node")) + } + node, err := aistack.FetchNodeStatus(ctx) + if err != nil { + fail(err) + } + fmt.Printf("healthy=true\nnode_memory=%d\n", node.Memory) + case len(os.Args) == 3 && os.Args[1] == "model": + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + node, err := aistack.FetchNodeStatus(ctx) + if err != nil { + fail(err) + } + live, err := aistack.ResolveFamilyLive(ctx, aistack.FamilyQwen, os.Args[2], aistack.DefaultFetch) + if err != nil { + fail(err) + } + ranked := aistack.RankCandidateModels(aistack.FilterEligibleModels(live, node.Memory)) + if len(ranked) == 0 { + fail(fmt.Errorf("no eligible file in %s for %d bytes of memory", os.Args[2], node.Memory)) + } + choice := ranked[0] + ref := choice.ModelRef() + fmt.Printf("ref=%s\nfile=%s\nquant=%s\nsize_bytes=%d\n", ref, choice.File, choice.Quant, choice.SizeBytes) + start := time.Now() + if err := aistack.PullModel(ctx, ref); err != nil { + fail(err) + } + fmt.Printf("pull_seconds=%.0f\n", time.Since(start).Seconds()) + if err := aistack.ConfigureActiveModel(ctx, ref); err != nil { + fail(err) + } + active, err := aistack.ReadActiveModel(ctx) + if err != nil { + fail(err) + } + fmt.Println("active=" + active) + if active != ref { + fail(fmt.Errorf("alias %s reads %q, want %q", aistack.ActiveModelAlias, active, ref)) + } + case len(os.Args) == 3 && os.Args[1] == "launch": state := troubleshoot.Detect() - // llmman is Agent Mode's install, not this feature's; any - // absolute executable satisfies Ready for an environment read. - if state.LLMManPath == "" { - state.LLMManPath = "/usr/bin/true" + printState(state) + profile := troubleshoot.ProfileAt(os.Args[2]) + if err := profile.Write(state.ServerPath); err != nil { + fail(err) + } + cmd, err := troubleshoot.Command(state, profile, aistack.ActiveModelAlias) + if err != nil { + fail(err) + } + fmt.Println("argv=" + strings.Join(cmd.Args, " ")) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + err = cmd.Run() + fmt.Printf("launch_exit=%v\n", err) + if err != nil { + os.Exit(1) } - cmd, err := troubleshoot.Command(state, troubleshoot.ProfileAt(os.Args[2]), "bluefin-active") + case len(os.Args) == 3 && os.Args[1] == "command-path": + cmd, err := troubleshoot.Command(troubleshoot.Detect(), troubleshoot.ProfileAt(os.Args[2]), aistack.ActiveModelAlias) if err != nil { fail(err) } @@ -739,7 +813,7 @@ spec: SRC=0 "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 || SRC=$? echo "${SRC}" >"${G}/setup-rc.txt" - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 || true + # (`command-path` runs after Agent Mode, once llmman really resolves.) { "${BREW}" list --versions linux-mcp-server cpio rpm2cpio "${BREW}" list --cask --versions goose-linux @@ -749,34 +823,118 @@ spec: exit 0 fi SERVER="$(sed -n 's/^server=//p' "${G}/setup.log")" - DESKTOP="$(sed -n 's/^desktop=//p' "${G}/setup.log")" ls -la "${HOME}/.config" >"${G}/home-config-before.txt" 2>&1 "${LANE}/bin/troubleshootcheck" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } cp "${PROFILE}/goose/config/config.yaml" "${G}/config.before.yaml" - ( - export GOOSE_PATH_ROOT="${PROFILE}/goose" XDG_CONFIG_HOME="${PROFILE}/desktop" - export GOOSE_PROVIDER=openai OPENAI_HOST=http://127.0.0.1:9 OPENAI_API_KEY=x GOOSE_MODEL=bluefin-active - export ELECTRON_OZONE_PLATFORM_HINT=wayland - exec "${DESKTOP}" --ozone-platform=wayland - ) >"${G}/goose-desktop.log" 2>&1 & - GOOSE_PID=$! - sleep 35 + # Agent Mode for real, through ChairLift's aistack: brew bundle, + # `llmman serve --pull-only`, the user unit, readiness on + # /llmman/node. Then the smallest Qwen3 through the alias path. + ARC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? + echo "${ARC}" >"${G}/agent-mode-rc.txt" + systemctl --user status chairlift-llmman.service --no-pager >"${G}/llmman-unit.txt" 2>&1 || true + if [ "${ARC}" -ne 0 ]; then + echo "agent-mode-failed" >"${G}/rc.txt" + exit 0 + fi + LLMMAN="$(sed -n 's/^llmman=//p' "${G}/agent-mode.log")" + MODEL_REPOS=(unsloth/Qwen3-0.6B-GGUF unsloth/Qwen3-1.7B-GGUF) + MRC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[0]}" >"${G}/model-1.log" 2>&1 || MRC=$? + if [ "${MRC}" -ne 0 ]; then + echo "model-failed" >"${G}/rc.txt" + exit 0 + fi + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 || true + + # The real launch: troubleshoot.Command(Detect(), profile, alias), + # i.e. `llmman launch goose-desktop --model bluefin-active` in the + # profile, from a PATH without Homebrew, no Electron hints. + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" launch "${DATA}" >"${G}/launch.log" 2>&1 & + LAUNCH_PID=$! + sleep 45 # Evidence while it runs: SingletonLock is removed on clean exit. + GOOSE_MAIN="" + for pid in $(pgrep -u "$(id -u)" -f '/usr/lib/Goose/Goose'); do + tr '\0' ' ' <"/proc/${pid}/cmdline" 2>/dev/null | grep -q -- '--type=' || { GOOSE_MAIN="${pid}"; break; } + done { - echo "alive=$(kill -0 "${GOOSE_PID}" 2>/dev/null && echo yes || echo no)" + echo "alive=$( [ -n "${GOOSE_MAIN}" ] && kill -0 "${GOOSE_MAIN}" 2>/dev/null && echo yes || echo no)" + echo "goose_main_pid=${GOOSE_MAIN}" echo "profile_lock=$(readlink "${PROFILE}/desktop/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" } >"${G}/lock.txt" - ps -eo pid,args | grep -i -E 'goose|electron' | grep -v grep >"${G}/processes.txt" || true + if [ -n "${GOOSE_MAIN}" ]; then + tr '\0' '\n' <"/proc/${GOOSE_MAIN}/environ" \ + | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|PATH|WAYLAND_DISPLAY|DISPLAY)=' >"${G}/goose-environ.txt" + tr '\0' ' ' <"/proc/${GOOSE_MAIN}/cmdline" >"${G}/goose-cmdline.txt" + fi + ps -eo pid,args | grep -i -E 'goose|electron|llmman' | grep -v grep | cut -c1-400 >"${G}/processes.txt" || true gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/goose-window.png" >"${G}/screenshot.txt" 2>&1 || true + pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null sleep 5 - pkill -TERM -P "${GOOSE_PID}" 2>/dev/null; kill -TERM "${GOOSE_PID}" 2>/dev/null - pkill -TERM -u "$(id -u)" -f 'goose-desktop|goosed|Goose' 2>/dev/null - sleep 5 - pkill -KILL -u "$(id -u)" -f 'goose-desktop|goosed|Goose' 2>/dev/null - wait "${GOOSE_PID}" 2>/dev/null + pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + wait "${LAUNCH_PID}" 2>/dev/null + + # A real answer with tools: the goose CLI through llmman in the same + # profile, so the same extensions and hints. + cat >"${G}/extract.py" <<'PYX' + import json, os, re, sqlite3, sys + root, out = sys.argv[1], sys.argv[2] + calls, responses, seen = [], [], set() + def walk(o): + if isinstance(o, dict): + kind = o.get("type") + if kind in ("toolRequest", "tool_request"): + call = o.get("toolCall") or o.get("tool_call") or {} + value = call.get("value", call) if isinstance(call, dict) else {} + key = (o.get("id"), json.dumps(value, sort_keys=True, default=str)) + if key not in seen: + seen.add(key) + calls.append({"id": o.get("id"), "name": value.get("name"), "arguments": value.get("arguments")}) + if kind in ("toolResponse", "tool_response"): + responses.append({"id": o.get("id"), "result": json.dumps(o.get("toolResult") or o.get("tool_result"), default=str)[:600]}) + for v in o.values(): + walk(v) + elif isinstance(o, list): + for v in o: + walk(v) + db = os.path.join(root, "data", "sessions", "sessions.db") + con = sqlite3.connect(db) + tables = [r[0] for r in con.execute("select name from sqlite_master where type='table'")] + for table in tables: + for row in con.execute(f'select * from "{table}"'): + for cell in row: + if isinstance(cell, str) and re.search(r"tool_?[Rr]e(quest|sponse)", cell): + try: + walk(json.loads(cell)) + except ValueError: + pass + names = [c["name"] or "" for c in calls] + json.dump({"tables": tables, "tool_calls": calls, "tool_responses": responses, + "linux_tools_called": any(n.startswith("linux-tools__") for n in names), + "search_knowledge_called": "bluefin-knowledge__search_knowledge" in names}, + open(out, "w"), indent=1, default=str) + PYX + PROMPT="Use the linux-tools extension to list failed systemd units on this machine, then use search_knowledge to search the Project Bluefin knowledge base for 'bootc update'. Report both results." + for attempt in 1 2; do + if [ "${attempt}" -eq 2 ]; then + grep -q '"linux_tools_called": true' "${G}/tool-calls-1.json" 2>/dev/null \ + && grep -q '"search_knowledge_called": true' "${G}/tool-calls-1.json" 2>/dev/null && break + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[1]}" >"${G}/model-2.log" 2>&1 || break + fi + CRC=0 + env GOOSE_PATH_ROOT="${PROFILE}/goose" XDG_CONFIG_HOME="${PROFILE}/desktop" \ + PATH="$(dirname "${BREW}"):/usr/bin:/bin" \ + timeout 900 "${LLMMAN}" launch goose --model bluefin-active -- run --text "${PROMPT}" \ + >"${G}/cli-${attempt}.log" 2>&1 "${G}/cli-${attempt}-rc.txt" + python3 "${G}/extract.py" "${PROFILE}/goose" "${G}/tool-calls-${attempt}.json" >"${G}/extract-${attempt}.log" 2>&1 || true + latest="$(ls -t "${PROFILE}/goose/state/logs/cli"/*/* "${PROFILE}/goose/state/logs/cli"/* 2>/dev/null | grep -v '/$' | head -1)" + [ -n "${latest}" ] && [ -f "${latest}" ] && cp "${latest}" "${G}/cli-${attempt}-goose.log" + done cp "${PROFILE}/goose/config/config.yaml" "${G}/config.after.yaml" 2>/dev/null "${LANE}/bin/troubleshootcheck" enabled "${PROFILE}/goose/config/config.yaml" >"${G}/enabled-extensions.txt" 2>&1 @@ -867,12 +1025,14 @@ spec: echo '{"status":"skipped"}' >"${G}/verdict.json" else echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - for f in setup.log setup-rc.txt command-env.txt brew-installed.txt; do + for f in setup.log setup-rc.txt brew-installed.txt agent-mode.log agent-mode-rc.txt llmman-unit.txt \ + model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ + cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do echo "--- ${f}" >&2 cat "${G}/${f}" >&2 2>/dev/null || true done GOOSE_STATE="${GOOSE_STATE}" python3 - <<'PY' || GOOSE_RC=$? - import json, os, sys + import json, os, re, sys g = "/tmp/results/goose" def read(name): try: @@ -890,28 +1050,51 @@ spec: setup = kv("setup.log") command_path = kv("command-env.txt").get("PATH", "") brew_bin = os.path.dirname(read("brew-path.txt").strip()) + agent = kv("agent-mode.log") + model = kv("model-2.log") if read("model-2.log") else kv("model-1.log") + desktop_model = kv("model-1.log") + environ = kv("goose-environ.txt") + attempt = "2" if read("cli-2-rc.txt") else "1" + try: + tools = json.loads(read(f"tool-calls-{attempt}.json")) + except ValueError: + tools = {} + profile = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting" checks = { "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", "isolation_run_completed": state == "done", "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), + "agent_mode_enabled_and_healthy": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", + "model_pulled_and_alias_set": bool(desktop_model.get("ref")) and desktop_model.get("active") == desktop_model.get("ref"), "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin and command_path.split(":")[1:] == ["/usr/bin", "/bin"], - "goose_was_running_at_35s": lock.get("alive") == "yes", + "llmman_launch_not_refused": lock.get("alive") == "yes" + and not re.search(r"^(Error|\[llmman\].*refus)", read("launch.log"), re.M), + "environ_goose_path_root": environ.get("GOOSE_PATH_ROOT") == profile + "/goose", + "environ_xdg_config_home": environ.get("XDG_CONFIG_HOME") == profile + "/desktop", + "environ_goose_provider_openai": environ.get("GOOSE_PROVIDER") == "openai", + "environ_openai_host_agent_mode": "127.0.0.1:17434" in environ.get("OPENAI_HOST", ""), + "environ_goose_model_is_resolved_ref": environ.get("GOOSE_MODEL") == desktop_model.get("ref"), "singleton_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", "no_singleton_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", + "cli_exit_zero": read(f"cli-{attempt}-rc.txt").strip() == "0", + "linux_tools_tool_called": tools.get("linux_tools_called") is True, + "search_knowledge_tool_called": tools.get("search_knowledge_called") is True, "only_linux_tools_and_bluefin_knowledge_enabled": enabled == ["bluefin-knowledge", "linux-tools"], "goose_data_or_state_under_path_root": any(p.startswith(root + "/data/") or p.startswith(root + "/state/") for p in tree), "nothing_in_home_config_goose": "ABSENT /home/bluefin-test/.config/goose" in paths, } verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, - "checks": checks, "lock": lock, "enabled": enabled} + "checks": checks, "lock": lock, "enabled": enabled, "cli_attempt": attempt, + "model": model, "desktop_model": desktop_model, "environ": environ, + "tool_calls": [c.get("name") for c in tools.get("tool_calls", [])]} json.dump(verdict, open(os.path.join(g, "verdict.json"), "w"), indent=1) print(json.dumps(verdict, indent=1)) print(paths) sys.exit(0 if verdict["status"] == "pass" else 1) PY - tail -n 60 "${G}/goose-desktop.log" >&2 2>/dev/null || true + tail -n 60 "${G}/launch.log" >&2 2>/dev/null || true fi echo "behave exit ${BEHAVE_RC}; suite verdict ${SUITE_RC}; goose verdict ${GOOSE_RC}" >&2 diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index 2492a433b8..b657df2e72 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -91,7 +91,9 @@ def test_wayland_only_never_x11(): runner = _heredoc(_runner(), "RUNNER") assert "GDK_BACKEND=wayland" in runner assert "unset DISPLAY XAUTHORITY" in runner - assert "ELECTRON_OZONE_PLATFORM_HINT=wayland" in runner + # Goose's window comes up exactly as ChairLift launches it: no Electron + # platform hint, and no DISPLAY to fall back to (unset above). + assert "ELECTRON_OZONE_PLATFORM_HINT" not in runner assert "org.gnome.Shell.Screenshot" in runner _bash_parses(runner) @@ -126,20 +128,34 @@ def test_goose_check_asserts_profile_isolation(): assert '"${LANE}/bin/troubleshootcheck" command-path' in runner assert 'eval "$("${BREW}" shellenv)"' not in runner assert '"${BREW}" install' not in runner - for setting in ( - 'GOOSE_PATH_ROOT="${PROFILE}/goose"', - 'XDG_CONFIG_HOME="${PROFILE}/desktop"', - "GOOSE_PROVIDER=openai", - "OPENAI_HOST=http://127.0.0.1:9", - "GOOSE_MODEL=bluefin-active", - ): - assert setting in runner, setting + # Agent Mode, the model, and the launch are ChairLift's own code paths. + for step in ("agent-mode", 'model "${MODEL_REPOS[0]}"', 'launch "${DATA}"'): + assert f'"${{LANE}}/bin/troubleshootcheck" {step}' in runner, step + assert "/usr/bin/true" not in _template()["initContainers"][0]["args"][0] + # Nothing fakes the provider: llmman hands Goose its environment. + for fake in ("GOOSE_PROVIDER=openai", "OPENAI_HOST=http://127.0.0.1:9", "OPENAI_API_KEY=x"): + assert fake not in runner, fake + assert 'launch goose --model bluefin-active -- run --text' in runner + build = _template()["initContainers"][0]["args"][0] + for call in ("aistack.Enable(ctx)", "aistack.WaitHealthy(", "aistack.PullModel(", "aistack.ConfigureActiveModel(", + "troubleshoot.Command(state, profile, aistack.ActiveModelAlias)"): + assert call in build, call source = _runner() for check in ( "setup_ran_without_brew_on_path", "chairlift_setup_exit_zero", "goose_desktop_resolves_after_setup", "command_env_path_has_brew_bin_first", + "agent_mode_enabled_and_healthy", + "model_pulled_and_alias_set", + "llmman_launch_not_refused", + "environ_goose_path_root", + "environ_xdg_config_home", + "environ_goose_provider_openai", + "environ_openai_host_agent_mode", + "environ_goose_model_is_resolved_ref", + "linux_tools_tool_called", + "search_knowledge_tool_called", "singleton_lock_in_profile", "no_singleton_lock_in_home_config", "only_linux_tools_and_bluefin_knowledge_enabled", From 7f4441699eab09601a2fb2c01b9339b4aeebe681 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 14:19:35 -0400 Subject: [PATCH 05/19] fix(argo): keep GNOME Shell's welcome dialog off the suite's keyboard Run chairlift-wayland-lane-2nlfh failed every keyboard scenario: the first-login "Welcome to Bluefin / Take Tour" dialog, a Shell modal, took the keys while the ChairLift window held focus (Main.modalCount was 1). Mark it shown in the test user's dconf before any session starts, and refuse to start the suite while any Shell modal is up. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 24 +++++++++++++++++++ .../unit/test_run_chairlift_wayland_tests.py | 8 +++++++ 2 files changed, 32 insertions(+) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 9e3c5ffcac..35b29f8a92 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -610,6 +610,16 @@ spec: systemctl status user@1000.service --no-pager --full >&2 || true exit 1 fi + # GNOME Shell's first-login "Welcome to Bluefin / Take Tour" dialog + # is a Shell modal: while it is up it takes every key, so the + # suite's input reaches no window even though the window holds + # focus (observed: a whole run of keyboard steps failed behind + # it). Mark it shown in the test user's own dconf before any + # session starts. + sudo -u bluefin-test env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + gsettings set org.gnome.shell welcome-dialog-last-shown-version "'999'" + sudo -u bluefin-test env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + gsettings get org.gnome.shell welcome-dialog-last-shown-version >&2 # qecore-headless (system interpreter; the target is disposable). python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --default-pip >&2 @@ -734,6 +744,20 @@ spec: echo 3 >"${OUT}/behave-rc.txt" exit 3 fi + # No Shell modal may hold the keyboard when the suite starts. + for attempt in $(seq 1 30); do + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.modalCount' 2>/dev/null | grep -q "(true, '0')" && break + sleep 1 + done + if ! gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.modalCount' 2>/dev/null | grep -q "(true, '0')"; then + echo "a GNOME Shell modal is still holding the keyboard" >&2 + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${OUT}/shell-modal.png" >&2 || true + echo 6 >"${OUT}/behave-rc.txt" + exit 6 + fi pgrep -u "$(id -u)" -f gnome-ponytail-daemon >/dev/null || { /usr/libexec/gnome-ponytail-daemon & sleep 2; } # Wayland only: the window must reach the session's compositor diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index b657df2e72..0a237b7f10 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -98,6 +98,14 @@ def test_wayland_only_never_x11(): _bash_parses(runner) +def test_no_shell_modal_holds_the_keyboard(): + source = _runner() + setup = _heredoc(source, "NESTED_SETUP") + assert "gsettings set org.gnome.shell welcome-dialog-last-shown-version" in setup + runner = _heredoc(source, "RUNNER") + assert "'Main.modalCount'" in runner + + def test_suite_isolation_matches_the_fixture_harness(): source = _runner() runner = _heredoc(source, "RUNNER") From 1efd4584b34135ead018f47fddb47a37e9c00a7c Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 14:34:14 -0400 Subject: [PATCH 06/19] fix(argo): leave the overview before gating on Shell modals The session opens in the overview, which counts as a Shell modal, so the new gate refused every run (chairlift-wayland-lane-25vxb: modalCount 1, screenshot shows the overview). Hide it first. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- argo/workflow-templates/run-chairlift-wayland-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 35b29f8a92..fce8f71e42 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -744,10 +744,12 @@ spec: echo 3 >"${OUT}/behave-rc.txt" exit 3 fi - # No Shell modal may hold the keyboard when the suite starts. + # No Shell modal may hold the keyboard when the suite starts. The + # session opens in the overview, which is itself a modal, so leave + # it first; what remains (the welcome dialog, a prompt) is a fault. for attempt in $(seq 1 30); do gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ - --method org.gnome.Shell.Eval 'Main.modalCount' 2>/dev/null | grep -q "(true, '0')" && break + --method org.gnome.Shell.Eval 'Main.overview.hide(); Main.modalCount' 2>/dev/null | grep -q "(true, '0')" && break sleep 1 done if ! gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ From fee915bc40312538e631d391606bab934a496829 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 15:00:22 -0400 Subject: [PATCH 07/19] fix(argo): record Agent Mode's untrusted-tap failure, then continue On stock Dakota aistack.Enable fails: Homebrew refuses llmmanorg/tap (chairlift-wayland-lane-jhcs8), and ChairLift offers no way to trust a tap nothing is installed from yet. Record that as a failing check (agent_mode_enabled_on_stock_host), then apply the remedy brew's message gives the user and enable Agent Mode again through ChairLift so the launch and tool-call checks still produce evidence. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index fce8f71e42..0b0c3eea20 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -856,8 +856,22 @@ spec: # Agent Mode for real, through ChairLift's aistack: brew bundle, # `llmman serve --pull-only`, the user unit, readiness on # /llmman/node. Then the smallest Qwen3 through the alias path. - ARC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? + SARC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode-stock.log" 2>&1 || SARC=$? + echo "${SARC}" >"${G}/agent-mode-stock-rc.txt" + cp "${G}/agent-mode-stock.log" "${G}/agent-mode.log" + ARC="${SARC}" + if [ "${SARC}" -ne 0 ] && grep -q 'untrusted tap llmmanorg/tap' "${G}/agent-mode-stock.log"; then + # Observed on stock Dakota: Homebrew refuses llmmanorg/tap, and + # ChairLift offers no way to trust a tap nothing is installed from + # yet. That failure stays recorded (agent_mode_enabled_on_stock_host) + # and fails the check. To still exercise everything after it, do + # what brew's own message tells the user to do in a terminal, then + # turn Agent Mode on again through ChairLift. + "${BREW}" trust llmmanorg/tap >"${G}/user-trust.log" 2>&1 || true + ARC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? + fi echo "${ARC}" >"${G}/agent-mode-rc.txt" systemctl --user status chairlift-llmman.service --no-pager >"${G}/llmman-unit.txt" 2>&1 || true if [ "${ARC}" -ne 0 ]; then @@ -1051,7 +1065,8 @@ spec: echo '{"status":"skipped"}' >"${G}/verdict.json" else echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - for f in setup.log setup-rc.txt brew-installed.txt agent-mode.log agent-mode-rc.txt llmman-unit.txt \ + for f in setup.log setup-rc.txt brew-installed.txt agent-mode-stock.log agent-mode-stock-rc.txt user-trust.log \ + agent-mode.log agent-mode-rc.txt llmman-unit.txt \ model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do echo "--- ${f}" >&2 @@ -1091,6 +1106,7 @@ spec: "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", "isolation_run_completed": state == "done", "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), + "agent_mode_enabled_on_stock_host": read("agent-mode-stock-rc.txt").strip() == "0", "agent_mode_enabled_and_healthy": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", "model_pulled_and_alias_set": bool(desktop_model.get("ref")) and desktop_model.get("active") == desktop_model.get("ref"), "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin From ba18177ddeb7390ac58181306b19274be60a883b Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 15:22:06 -0400 Subject: [PATCH 08/19] fix(argo): prove Agent Mode on the stock host; test user is an administrator ChairLift now trusts only the llmman formula when Agent Mode installs it (chairlift 836f58e), so the lane's manual brew trust is gone: Agent Mode must enable on the stock host, and the check records that the formula is trusted while llmmanorg/tap is not. The test user joins wheel, as Bluefin's first user does. Without it, Agent Mode's Jan Flatpak failed with "Flatpak system operation Deploy not allowed for user" (chairlift-wayland-lane-q5xpf). Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 54 ++++++++++--------- .../unit/test_run_chairlift_wayland_tests.py | 6 ++- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 0b0c3eea20..78e30ee2e3 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -543,14 +543,19 @@ spec: fi printf '%s\n' "$1" >>/etc/group } - for group in video render input; do + # wheel: Bluefin's first user is an administrator, and the distro's + # flatpak polkit rules let wheel install system Flatpaks without a + # prompt. Agent Mode's Brewfile installs Jan that way; without wheel + # it failed with "Flatpak system operation Deploy not allowed for + # user" (chairlift-wayland-lane-q5xpf), which no desktop user sees. + for group in video render input wheel; do if ! grep -q "^${group}:" /etc/group; then if ! groupadd -r "${group}" 2>/dev/null; then nss_entry=$(getent group "${group}" || true) if [ -n "${nss_entry}" ]; then append_group_line "${nss_entry}" else - case "${group}" in video) gid=39 ;; render) gid=989 ;; *) gid=999 ;; esac + case "${group}" in video) gid=39 ;; render) gid=989 ;; wheel) gid=10 ;; *) gid=999 ;; esac append_group_line "${group}:x:${gid}:" fi fi @@ -558,9 +563,9 @@ spec: grep -q "^${group}:" /etc/group || { echo "Failed to create local group ${group}" >&2; exit 1; } done if ! id bluefin-test >/dev/null 2>&1; then - useradd -m -u 1000 -G video,render,input bluefin-test + useradd -m -u 1000 -G video,render,input,wheel bluefin-test else - usermod -aG video,render,input bluefin-test + usermod -aG video,render,input,wheel bluefin-test fi # Synthetic input: the container's own /dev/uinput, given to the # lane-local input group (see run-container-tests). @@ -853,26 +858,19 @@ spec: "${LANE}/bin/troubleshootcheck" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } cp "${PROFILE}/goose/config/config.yaml" "${G}/config.before.yaml" - # Agent Mode for real, through ChairLift's aistack: brew bundle, + # Agent Mode for real, through ChairLift's aistack on the stock host: + # tap llmmanorg/tap, trust only the llmman formula, brew bundle, # `llmman serve --pull-only`, the user unit, readiness on - # /llmman/node. Then the smallest Qwen3 through the alias path. - SARC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode-stock.log" 2>&1 || SARC=$? - echo "${SARC}" >"${G}/agent-mode-stock-rc.txt" - cp "${G}/agent-mode-stock.log" "${G}/agent-mode.log" - ARC="${SARC}" - if [ "${SARC}" -ne 0 ] && grep -q 'untrusted tap llmmanorg/tap' "${G}/agent-mode-stock.log"; then - # Observed on stock Dakota: Homebrew refuses llmmanorg/tap, and - # ChairLift offers no way to trust a tap nothing is installed from - # yet. That failure stays recorded (agent_mode_enabled_on_stock_host) - # and fails the check. To still exercise everything after it, do - # what brew's own message tells the user to do in a terminal, then - # turn Agent Mode on again through ChairLift. - "${BREW}" trust llmmanorg/tap >"${G}/user-trust.log" 2>&1 || true - ARC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? - fi + # /llmman/node. Nothing here trusts anything on ChairLift's behalf. + # Then the smallest Qwen3 through the alias path. + ARC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? echo "${ARC}" >"${G}/agent-mode-rc.txt" + # What Homebrew now trusts: the formula, not the whole tap. + "${BREW}" tap-info --json llmmanorg/tap >"${G}/tap-info.json" 2>"${G}/tap-info.err" || true + find "${HOME}" -maxdepth 3 -name trust.json -path '*homebrew*' 2>/dev/null | while read -r f; do + echo "== ${f}"; cat "${f}"; echo + done >"${G}/trust-state.txt" systemctl --user status chairlift-llmman.service --no-pager >"${G}/llmman-unit.txt" 2>&1 || true if [ "${ARC}" -ne 0 ]; then echo "agent-mode-failed" >"${G}/rc.txt" @@ -1065,7 +1063,7 @@ spec: echo '{"status":"skipped"}' >"${G}/verdict.json" else echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - for f in setup.log setup-rc.txt brew-installed.txt agent-mode-stock.log agent-mode-stock-rc.txt user-trust.log \ + for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt \ agent-mode.log agent-mode-rc.txt llmman-unit.txt \ model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do @@ -1100,14 +1098,20 @@ spec: tools = json.loads(read(f"tool-calls-{attempt}.json")) except ValueError: tools = {} + try: + tap_info = json.loads(read("tap-info.json")) + tap_trusted = tap_info[0].get("trusted") if tap_info else None + except (ValueError, IndexError, AttributeError): + tap_trusted = None profile = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting" checks = { "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", "isolation_run_completed": state == "done", "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), - "agent_mode_enabled_on_stock_host": read("agent-mode-stock-rc.txt").strip() == "0", - "agent_mode_enabled_and_healthy": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", + "agent_mode_enabled_on_stock_host": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", + "llmman_formula_trusted": "llmmanorg/tap/llmman" in read("trust-state.txt"), + "llmmanorg_tap_not_trusted": tap_trusted is False, "model_pulled_and_alias_set": bool(desktop_model.get("ref")) and desktop_model.get("active") == desktop_model.get("ref"), "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin and command_path.split(":")[1:] == ["/usr/bin", "/bin"], diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index 0a237b7f10..cc48620c16 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -144,6 +144,8 @@ def test_goose_check_asserts_profile_isolation(): for fake in ("GOOSE_PROVIDER=openai", "OPENAI_HOST=http://127.0.0.1:9", "OPENAI_API_KEY=x"): assert fake not in runner, fake assert 'launch goose --model bluefin-active -- run --text' in runner + # Agent Mode must work on the stock host: the lane trusts nothing itself. + assert '"${BREW}" trust' not in runner and "brew trust" not in runner build = _template()["initContainers"][0]["args"][0] for call in ("aistack.Enable(ctx)", "aistack.WaitHealthy(", "aistack.PullModel(", "aistack.ConfigureActiveModel(", "troubleshoot.Command(state, profile, aistack.ActiveModelAlias)"): @@ -154,7 +156,9 @@ def test_goose_check_asserts_profile_isolation(): "chairlift_setup_exit_zero", "goose_desktop_resolves_after_setup", "command_env_path_has_brew_bin_first", - "agent_mode_enabled_and_healthy", + "agent_mode_enabled_on_stock_host", + "llmman_formula_trusted", + "llmmanorg_tap_not_trusted", "model_pulled_and_alias_set", "llmman_launch_not_refused", "environ_goose_path_root", From b3c1cd8235a827c5e25ffe20711fd9d3c344b08c Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 15:53:56 -0400 Subject: [PATCH 09/19] fix(argo): run ChairLift's steps as app units in the user's session The runner script sits in podman's exec cgroup, outside any logind session, so polkit treated ChairLift's Agent Mode install of Jan (a system Flatpak) as coming from an inactive subject and refused it even for a wheel member (chairlift-wayland-lane-w7wtx, -nh4wz). Run every ChairLift step the way GNOME runs an app: a transient unit in the user manager's app.slice. Record logind and pkcheck evidence for both contexts. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 78e30ee2e3..57928499aa 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -543,11 +543,11 @@ spec: fi printf '%s\n' "$1" >>/etc/group } - # wheel: Bluefin's first user is an administrator, and the distro's - # flatpak polkit rules let wheel install system Flatpaks without a - # prompt. Agent Mode's Brewfile installs Jan that way; without wheel - # it failed with "Flatpak system operation Deploy not allowed for - # user" (chairlift-wayland-lane-q5xpf), which no desktop user sees. + # wheel: Bluefin's first user is an administrator. Dakota's + # org.freedesktop.Flatpak.rules lets an active, local wheel member + # install system Flatpaks without a prompt, and Agent Mode's + # Brewfile installs Jan that way ("Flatpak system operation Deploy + # not allowed for user" without it, chairlift-wayland-lane-q5xpf). for group in video render input wheel; do if ! grep -q "^${group}:" /etc/group; then if ! groupadd -r "${group}" 2>/dev/null; then @@ -832,6 +832,10 @@ spec: BREW="$(cat "${G}/brew-path.txt")" DATA=/home/bluefin-test/goose-isolation PROFILE="${DATA}/chairlift/troubleshooting" + # Every ChairLift step runs the way GNOME runs an app it launched: a + # transient unit in the user manager's app.slice, inside the user's + # logind context. (This script itself sits in podman's exec cgroup, + # outside any session, so polkit would treat it as inactive.) # The install goes through ChairLift itself — troubleshoot.Setup over # internal/homebrew — with a PATH that deliberately lacks Homebrew's # bin and no `brew shellenv`, which is how a direct `chairlift` launch @@ -839,10 +843,11 @@ spec: # brew's children Homebrew's bin itself: the goose-linux cask's # preflight runs a bare `rpm2cpio | cpio` (exit 127 without it). # Auto-update is off only to bound the lane's time. - BARE_ENV=(env -u HOMEBREW_PREFIX -u HOMEBREW_CELLAR -u HOMEBREW_REPOSITORY - PATH=/usr/bin:/bin HOMEBREW_NO_AUTO_UPDATE=1) + BARE_ENV=(systemd-run --user --wait --pipe --quiet --collect --slice=app.slice + -p "UnsetEnvironment=HOMEBREW_PREFIX HOMEBREW_CELLAR HOMEBREW_REPOSITORY" + --setenv=PATH=/usr/bin:/bin --setenv=HOMEBREW_NO_AUTO_UPDATE=1) SRC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 || SRC=$? + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 "${G}/setup-rc.txt" # (`command-path` runs after Agent Mode, once llmman really resolves.) { @@ -863,8 +868,24 @@ spec: # `llmman serve --pull-only`, the user unit, readiness on # /llmman/node. Nothing here trusts anything on ChairLift's behalf. # Then the smallest Qwen3 through the alias path. + # Where this process stands with logind and polkit: Agent Mode's Jan + # Flatpak is a system install, which Dakota's flatpak rule allows + # only for an active, local wheel member. + { + id + echo "cgroup: $(cat /proc/self/cgroup)" + loginctl list-sessions --no-legend + loginctl show-user "$(id -u)" -p Display -p Sessions -p State + for s in $(loginctl list-sessions --no-legend | awk '{print $1}'); do + loginctl show-session "${s}" -p Id -p Name -p Seat -p Type -p Class -p Active -p Remote -p State + done + echo "pkcheck (this script, podman exec):" + pkcheck --action-id org.freedesktop.Flatpak.app-install --process "${BASHPID}" 2>&1; echo "rc=$?" + echo "pkcheck (app unit, as ChairLift's steps run):" + "${BARE_ENV[@]}" bash -c 'cat /proc/self/cgroup; pkcheck --action-id org.freedesktop.Flatpak.app-install --process "${BASHPID}" 2>&1; echo rc=$?' &1 + } >"${G}/polkit-session.txt" 2>&1 ARC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 || ARC=$? + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 "${G}/agent-mode-rc.txt" # What Homebrew now trusts: the formula, not the whole tap. "${BREW}" tap-info --json llmmanorg/tap >"${G}/tap-info.json" 2>"${G}/tap-info.err" || true @@ -879,17 +900,17 @@ spec: LLMMAN="$(sed -n 's/^llmman=//p' "${G}/agent-mode.log")" MODEL_REPOS=(unsloth/Qwen3-0.6B-GGUF unsloth/Qwen3-1.7B-GGUF) MRC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[0]}" >"${G}/model-1.log" 2>&1 || MRC=$? + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[0]}" >"${G}/model-1.log" 2>&1 "${G}/rc.txt" exit 0 fi - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 || true + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 "${G}/launch.log" 2>&1 & + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" launch "${DATA}" >"${G}/launch.log" 2>&1 /dev/null \ && grep -q '"search_knowledge_called": true' "${G}/tool-calls-1.json" 2>/dev/null && break - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[1]}" >"${G}/model-2.log" 2>&1 || break + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[1]}" >"${G}/model-2.log" 2>&1 "${G}/verdict.json" else echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt \ + for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt polkit-session.txt \ agent-mode.log agent-mode-rc.txt llmman-unit.txt \ model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do From 5a109d72ce8871ea313b58496d77bcf2009a0ea6 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 16:07:47 -0400 Subject: [PATCH 10/19] fix(argo): give ChairLift's app units a desktop's task headroom Running Setup as an app unit hit the user slice's pids limit: brew's download threads failed with "can't create Thread: Resource temporarily unavailable" (chairlift-wayland-lane-dsjsw). Lift TasksMax on the user slice, app.slice, and each unit, and record the limits. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 57928499aa..1a6f12f470 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -608,6 +608,9 @@ spec: systemctl mask --runtime brew-update.timer brew-upgrade.timer >/dev/null 2>&1 || true loginctl enable-linger bluefin-test + # logind sizes the user slice's TasksMax from the container's pids + # budget; a desktop has orders of magnitude more headroom. + systemctl set-property --runtime user-1000.slice TasksMax=infinity || true for _ in $(seq 1 30); do [ -S /run/user/1000/bus ] && break; sleep 1; done if [ ! -S /run/user/1000/bus ]; then echo "user@1000.service did not create /run/user/1000/bus after enable-linger" >&2 @@ -843,9 +846,15 @@ spec: # brew's children Homebrew's bin itself: the goose-linux cask's # preflight runs a bare `rpm2cpio | cpio` (exit 127 without it). # Auto-update is off only to bound the lane's time. - BARE_ENV=(systemd-run --user --wait --pipe --quiet --collect --slice=app.slice + # TasksMax: the user manager's default is a share of the container's + # pids budget, far below a desktop's; brew's download threads hit it + # ("can't create Thread", chairlift-wayland-lane-dsjsw). + BARE_ENV=(systemd-run --user --wait --pipe --quiet --collect --slice=app.slice -p TasksMax=infinity -p "UnsetEnvironment=HOMEBREW_PREFIX HOMEBREW_CELLAR HOMEBREW_REPOSITORY" --setenv=PATH=/usr/bin:/bin --setenv=HOMEBREW_NO_AUTO_UPDATE=1) + systemctl --user set-property --runtime app.slice TasksMax=infinity 2>/dev/null || true + for unit in user-1000.slice; do systemctl show "${unit}" -p TasksMax -p TasksCurrent; done >"${G}/tasks-limits.txt" 2>&1 + systemctl --user show app.slice -p TasksMax -p TasksCurrent >>"${G}/tasks-limits.txt" 2>&1 SRC=0 "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 "${G}/setup-rc.txt" From a5b8891fed4b922dfcacb38f4447967a20412cc5 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 16:23:35 -0400 Subject: [PATCH 11/19] fix(argo): record what llmman stored, and keep going past aistack's check aistack.PullModel reported unsloth/Qwen3-0.6B-GGUF:Q4_K_M "not found in daemon stored models" after the pull (chairlift-wayland-lane-rx4k2). Record /llmman/node, llmman ls and the installed Flatpaks; when only that check disagrees, set the alias through ChairLift's ConfigureActiveModel so the launch and tool checks still run, while model_pulled_and_alias_set stays false. Fix the pkcheck pid inside systemd-run, which expands ${...} itself. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 1a6f12f470..fb46d830f0 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -274,6 +274,17 @@ spec: if active != ref { fail(fmt.Errorf("alias %s reads %q, want %q", aistack.ActiveModelAlias, active, ref)) } + case len(os.Args) == 3 && os.Args[1] == "alias": + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := aistack.ConfigureActiveModel(ctx, os.Args[2]); err != nil { + fail(err) + } + active, err := aistack.ReadActiveModel(ctx) + if err != nil { + fail(err) + } + fmt.Println("active=" + active) case len(os.Args) == 3 && os.Args[1] == "launch": state := troubleshoot.Detect() printState(state) @@ -891,7 +902,7 @@ spec: echo "pkcheck (this script, podman exec):" pkcheck --action-id org.freedesktop.Flatpak.app-install --process "${BASHPID}" 2>&1; echo "rc=$?" echo "pkcheck (app unit, as ChairLift's steps run):" - "${BARE_ENV[@]}" bash -c 'cat /proc/self/cgroup; pkcheck --action-id org.freedesktop.Flatpak.app-install --process "${BASHPID}" 2>&1; echo rc=$?' &1 + "${BARE_ENV[@]}" bash -c 'cat /proc/self/cgroup; pkcheck --action-id org.freedesktop.Flatpak.app-install --process "$(cut -d" " -f4 /proc/self/stat)" 2>&1; echo rc=$?' &1 } >"${G}/polkit-session.txt" 2>&1 ARC=0 "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 "${G}/model-1.log" 2>&1 "${G}/node.json" 2>&1 || true + "${LLMMAN}" ls >"${G}/llmman-ls.txt" 2>&1 || true + flatpak list --app --columns=application,installation >"${G}/flatpaks.txt" 2>&1 || true if [ "${MRC}" -ne 0 ]; then - echo "model-failed" >"${G}/rc.txt" - exit 0 + if ! grep -q 'not found in daemon stored models' "${G}/model-1.log"; then + echo "model-failed" >"${G}/rc.txt" + exit 0 + fi + # The pull itself completed; only aistack's stored-model check + # disagreed with llmman (model_pulled_and_alias_set stays false and + # fails the check). Set the alias through ChairLift's own + # ConfigureActiveModel so the launch and tool checks still run. + REF="$(sed -n 's/^ref=//p' "${G}/model-1.log")" + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" alias "${REF}" >"${G}/alias-1.log" 2>&1 "${G}/rc.txt" + exit 0 + } fi "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 /dev/null \ && grep -q '"search_knowledge_called": true' "${G}/tool-calls-1.json" 2>/dev/null && break - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[1]}" >"${G}/model-2.log" 2>&1 "${G}/model-2.log" 2>&1 "${G}/alias-2.log" 2>&1 &2 for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt polkit-session.txt \ - agent-mode.log agent-mode-rc.txt llmman-unit.txt \ + agent-mode.log agent-mode-rc.txt llmman-unit.txt flatpaks.txt node.json llmman-ls.txt alias-1.log alias-2.log \ model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do echo "--- ${f}" >&2 From 5be927860611774e617358e3b8d1e4a6245c6049 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 16:34:50 -0400 Subject: [PATCH 12/19] fix(argo): read the Goose browser process's environment through sudo Its /proc environ read back empty as the test user (chairlift-wayland-lane-jhxkh), so every environ check failed on no data. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- argo/workflow-templates/run-chairlift-wayland-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index fb46d830f0..e92eb326f7 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -960,7 +960,9 @@ spec: echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" } >"${G}/lock.txt" if [ -n "${GOOSE_MAIN}" ]; then - tr '\0' '\n' <"/proc/${GOOSE_MAIN}/environ" \ + # The Electron browser process's environ read back empty as the + # test user (chairlift-wayland-lane-jhxkh); read it through sudo. + sudo -n cat "/proc/${GOOSE_MAIN}/environ" | tr '\0' '\n' \ | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|PATH|WAYLAND_DISPLAY|DISPLAY)=' >"${G}/goose-environ.txt" tr '\0' ' ' <"/proc/${GOOSE_MAIN}/cmdline" >"${G}/goose-cmdline.txt" fi From 0b2ab3ee44c214bb7d02a80b3cbb6feeff65ba39 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 16:55:06 -0400 Subject: [PATCH 13/19] fix(argo): read Goose's model environment from its goose serve backend The Electron browser process's environ came back without any of the provider names (chairlift-wayland-lane-t54w6). Record its raw contents and assert on the environment Goose hands its own backend. Document the end-to-end Goose check and the session/polkit and TasksMax traps. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 17 +++++--- docs/reference/workflow-reference.md | 42 +++++++++++++++---- docs/skills/argo-workflows/patterns.md | 17 ++++++++ 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index e92eb326f7..d7a4f203d5 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -960,10 +960,17 @@ spec: echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" } >"${G}/lock.txt" if [ -n "${GOOSE_MAIN}" ]; then - # The Electron browser process's environ read back empty as the - # test user (chairlift-wayland-lane-jhxkh); read it through sudo. - sudo -n cat "/proc/${GOOSE_MAIN}/environ" | tr '\0' '\n' \ - | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|PATH|WAYLAND_DISPLAY|DISPLAY)=' >"${G}/goose-environ.txt" + # The Electron browser process's environ came back without any of + # these names, also through sudo (chairlift-wayland-lane-jhxkh, -t54w6); + # record what it does hold, and read the environment the app + # hands its own `goose serve` backend, which is what talks to the + # model. + sudo -n cat "/proc/${GOOSE_MAIN}/environ" | tr '\0' '\n' | cut -c1-120 | head -40 >"${G}/goose-main-environ-raw.txt" + GOOSED="$(pgrep -u "$(id -u)" -f '/usr/lib/Goose/resources/bin/goose serve' | head -1)" + echo "goosed_pid=${GOOSED}" >>"${G}/lock.txt" + [ -n "${GOOSED}" ] && tr '\0' '\n' <"/proc/${GOOSED}/environ" \ + | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|PATH|WAYLAND_DISPLAY|DISPLAY)=' \ + | grep -v -E '^GOOSE_SERVER__SECRET_KEY=' >"${G}/goose-environ.txt" tr '\0' ' ' <"/proc/${GOOSE_MAIN}/cmdline" >"${G}/goose-cmdline.txt" fi ps -eo pid,args | grep -i -E 'goose|electron|llmman' | grep -v grep | cut -c1-400 >"${G}/processes.txt" || true @@ -1127,7 +1134,7 @@ spec: echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt polkit-session.txt \ agent-mode.log agent-mode-rc.txt llmman-unit.txt flatpaks.txt node.json llmman-ls.txt alias-1.log alias-2.log \ - model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-cmdline.txt \ + model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-main-environ-raw.txt goose-cmdline.txt \ cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do echo "--- ${f}" >&2 cat "${G}/${f}" >&2 2>/dev/null || true diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index f5ea389782..54dd4c3174 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -154,15 +154,39 @@ before retrying. activates it through `Shell.Eval` (needs `--unsafe-mode`). Dakota's `brew-preinstall.service` and brew timers are masked so they cannot hold Homebrew's locks during the Goose check. -- **Goose check:** ChairLift installs the packages itself - (`troubleshoot.Setup`: tap, `linux-mcp-server`, `cpio`, the `goose-linux` - cask) with `PATH=/usr/bin:/bin` and no `brew shellenv`, as a direct - `chairlift` launch would; the cask's bare `rpm2cpio | cpio` preflight - exits 127 unless ChairLift puts Homebrew's bin on brew's PATH. The check - then asserts `goose-desktop` resolves, `Command`'s PATH starts with - Homebrew's bin, and Goose launched in the profile keeps its lock, data, - state, and config there with only `linux-tools` and `bluefin-knowledge` - enabled. +- **Goose check (end to end, ChairLift's own code throughout):** + 1. `troubleshoot.Setup` installs the tap, `linux-mcp-server`, `cpio`, and + the `goose-linux` cask with `PATH=/usr/bin:/bin` and no + `brew shellenv`, as a direct `chairlift` launch would. The cask's bare + `rpm2cpio | cpio` preflight exits 127 unless ChairLift puts Homebrew's + bin on brew's PATH. + 2. `aistack.Enable` turns Agent Mode on, on the stock host. The check + records that `brew trust` covers the `llmman` formula and not + `llmmanorg/tap`. + 3. The smallest Qwen3 is resolved through aistack's live resolver, pulled, + and set as the `bluefin-active` alias. The lane also records + `/llmman/node` and `llmman ls`. + 4. The real `troubleshoot.Command` launch + (`llmman launch goose-desktop`) runs in the Wayland session. The check + reads the Goose process's environment (`GOOSE_PATH_ROOT`, + `XDG_CONFIG_HOME`, `GOOSE_PROVIDER=openai`, `OPENAI_HOST` on + `127.0.0.1:17434`, `GOOSE_MODEL`), checks where its lock is, and takes + a screenshot. + 5. `llmman launch goose -- run --text …`, run in the same profile, must + call a `linux-tools__*` tool and `bluefin-knowledge__search_knowledge`, + read back from `data/sessions/sessions.db`. If the 0.6B model will not + call tools, the lane retries once with the next size up. + + Throughout, the profile's lock, data, state, and config must stay inside + it, and only `linux-tools` and `bluefin-knowledge` may be enabled. +- **ChairLift runs as an app unit:** every ChairLift step runs through + `systemd-run --user --slice=app.slice`, the way GNOME starts an app. The + script itself runs in podman's exec cgroup, outside any logind session, + where polkit answers `auth_admin` to the system Flatpak install Agent Mode + does; inside an app unit, an active `wheel` member gets `yes`. The test + user joins `wheel`, as Bluefin's first user does. `TasksMax` is lifted on + the user slice and on the units, because brew's download threads hit the + container-sized default. - **Verdicts:** fails when behave fails, when 0 scenarios executed, or when any Goose check fails. Outputs `result`, `failed-scenarios`, and `goose-isolation` (JSON). Everything under `/tmp/results` (behave log, diff --git a/docs/skills/argo-workflows/patterns.md b/docs/skills/argo-workflows/patterns.md index 79f81b9844..3970c788b8 100644 --- a/docs/skills/argo-workflows/patterns.md +++ b/docs/skills/argo-workflows/patterns.md @@ -159,6 +159,23 @@ ghost: Screenshots come from `org.gnome.Shell.Screenshot.Screenshot`, which `--unsafe-mode` also opens to the test user; there is no root window for `xwd`. +Two more traps, from driving the app's own privileged-adjacent paths: + +- **`podman exec` is not in the user's session.** A process started with + `podman exec` sits in the target's exec cgroup, so logind assigns it to no + session and polkit answers `auth_admin` to actions it allows an active + local `wheel` member without a prompt (a system Flatpak install, for one). + Run app-side steps the way GNOME starts an app: + `systemd-run --user --wait --pipe --slice=app.slice`. `pkcheck` from each + context is the evidence. To hand `pkcheck` a PID, read it from + `/proc/self/stat`. `systemd-run` expands `${VAR}` in its command line + itself, and in the qecore-launched script `$$` reached `pkcheck` as a bare + `$` (the cause was not isolated). +- **The user slice's `TasksMax` is container-sized.** Inside a unit, brew's + download threads failed with "can't create Thread: Resource temporarily + unavailable". Lift `TasksMax` on `user-1000.slice`, `app.slice`, and the + unit itself. + ## Concurrency and scheduling ### VM concurrency: k8s native scheduling, no semaphores From 8ae35173ca8a6b66f2eb24202986b8cda2327727 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 17:25:39 -0400 Subject: [PATCH 14/19] feat(argo): final ChairLift lane shape for --ask-bluefin - Build: run ChairLift's `make ci` (golangci-lint v2.12.2 by checksum) on the untouched checkout and record it; stage chairlift and chairlift-wrapper as the Homebrew cask lays them out. - Behave: export the session's Wayland socket as an absolute path (the suite's MutterInputBackend and per-scenario runtime dirs need it); stop starting gnome-ponytail-daemon. - Engine: aistack.PullModel must succeed on its own (no alias fallback); launch Goose through a cold `chairlift --ask-bluefin` and through the distro menu's exact command (chairlift-wrapper via bash -c) with GNOME Shell's environment and a PATH without Homebrew, asserting the Goose backend's environment, lock, and the same outcome for both; check the CLI answer against systemctl --failed for 0.6B, 1.7B, then 4B. - Screenshots: ChairLift's own capture_walkthrough.sh under its wayland_session.sh (headless Mutter, 900x700), byproducts stripped. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 447 +++++++++++------- .../unit/test_run_chairlift_wayland_tests.py | 66 ++- 2 files changed, 327 insertions(+), 186 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index d7a4f203d5..762ee0ac96 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -46,10 +46,11 @@ spec: nodeSelector: node-role.kubernetes.io/control-plane: "true" kubernetes.io/hostname: ghost - # Build (~5m) + pull (≤34m worst case, normally minutes) + boot and pip - # (~5m) + one application launch per scenario (~25m, the suite's own - # budget) + the Goose check's network installs (~10m). - activeDeadlineSeconds: 5400 + # `make ci` and the build (~15m) + pull (≤34m worst case, normally + # minutes) + boot and pip (~5m) + one application launch per scenario + # (~12m) + the engine check: installs, Agent Mode, two launches, and up + # to three model pulls and CLI answers (≤15m each), and the screenshots. + activeDeadlineSeconds: 7200 # Leaf template holding the shared ghost QA capacity key, like # run-container-tests: each run boots one nested GNOME target. synchronization: @@ -96,10 +97,10 @@ spec: resources: requests: cpu: "2" - memory: 2Gi + memory: 4Gi limits: cpu: "4" - memory: 4Gi + memory: 8Gi volumeMounts: - mountPath: /workspace name: workspace @@ -117,7 +118,32 @@ spec: cd /workspace/chairlift mkdir -p /workspace/lane/bin /workspace/lane/inventory git rev-parse HEAD | tee /workspace/lane/chairlift-commit + + # `make ci`, ChairLift's whole host-independent CI gate, on the + # untouched checkout (before any lane tool is written into it). Its + # result is recorded and reported; the lane goes on either way, so + # one run answers both questions. golangci-lint is the version CI + # pins (.github/workflows/test.yml), fetched by checksum. + LINT_VERSION=2.12.2 + LINT_SHA256=8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 + LINT_TGZ="golangci-lint-${LINT_VERSION}-linux-amd64.tar.gz" + curl -sSfL -o "/tmp/${LINT_TGZ}" \ + "https://github.com/golangci/golangci-lint/releases/download/v${LINT_VERSION}/${LINT_TGZ}" + echo "${LINT_SHA256} /tmp/${LINT_TGZ}" | sha256sum -c - + tar -xzf "/tmp/${LINT_TGZ}" -C /tmp + install -m 0755 "/tmp/golangci-lint-${LINT_VERSION}-linux-amd64/golangci-lint" /usr/local/bin/golangci-lint + CI_RC=0 + make ci >/workspace/lane/make-ci.log 2>&1 || CI_RC=$? + echo "${CI_RC}" >/workspace/lane/make-ci-rc + echo "make ci exit ${CI_RC}"; tail -n 40 /workspace/lane/make-ci.log + git status --porcelain + CGO_ENABLED=0 make build-e2e schemas + # What a Homebrew install puts on disk: the GUI and the wrapper the + # desktop entry and the distro's Ask Bluefin menu entry run. + mkdir -p /workspace/lane/install + install -m 0755 build/chairlift /workspace/lane/install/chairlift + install -m 0755 data/chairlift-wrapper.sh /workspace/lane/install/chairlift-wrapper # Lane tools live inside the module so they may import internal/. # navjson: the page and shortcut inventories the Go gate @@ -178,8 +204,6 @@ spec: # ranks it, then aistack.PullModel and # aistack.ConfigureActiveModel (the alias path). # command-path the PATH troubleshoot.Command hands the launch. - # launch writes the profile and runs the real - # troubleshoot.Command(Detect(), profile, alias). # write troubleshoot.ProfileAt(dir).Write(server). # enabled the extensions a config.yaml leaves enabled. cat >build/lane/troubleshootcheck/main.go <<'GO' @@ -274,35 +298,6 @@ spec: if active != ref { fail(fmt.Errorf("alias %s reads %q, want %q", aistack.ActiveModelAlias, active, ref)) } - case len(os.Args) == 3 && os.Args[1] == "alias": - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - if err := aistack.ConfigureActiveModel(ctx, os.Args[2]); err != nil { - fail(err) - } - active, err := aistack.ReadActiveModel(ctx) - if err != nil { - fail(err) - } - fmt.Println("active=" + active) - case len(os.Args) == 3 && os.Args[1] == "launch": - state := troubleshoot.Detect() - printState(state) - profile := troubleshoot.ProfileAt(os.Args[2]) - if err := profile.Write(state.ServerPath); err != nil { - fail(err) - } - cmd, err := troubleshoot.Command(state, profile, aistack.ActiveModelAlias) - if err != nil { - fail(err) - } - fmt.Println("argv=" + strings.Join(cmd.Args, " ")) - cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr - err = cmd.Run() - fmt.Printf("launch_exit=%v\n", err) - if err != nil { - os.Exit(1) - } case len(os.Args) == 3 && os.Args[1] == "command-path": cmd, err := troubleshoot.Command(troubleshoot.Detect(), troubleshoot.ProfileAt(os.Args[2]), aistack.ActiveModelAlias) if err != nil { @@ -344,7 +339,7 @@ spec: fmt.Println(name) } default: - fmt.Fprintln(os.Stderr, "usage: troubleshootcheck setup | command-path | write | enabled ") + fmt.Fprintln(os.Stderr, "usage: troubleshootcheck setup | agent-mode | model | command-path | write | enabled ") os.Exit(2) } } @@ -779,7 +774,7 @@ spec: echo 6 >"${OUT}/behave-rc.txt" exit 6 fi - pgrep -u "$(id -u)" -f gnome-ponytail-daemon >/dev/null || { /usr/libexec/gnome-ponytail-daemon & sleep 2; } + # Wayland only: the window must reach the session's compositor # natively, never through the session's on-demand Xwayland. @@ -798,6 +793,10 @@ spec: echo 4 >"${OUT}/behave-rc.txt" exit 4 fi + # Absolute, as run_atspi.sh's wayland_session.sh exports it: each + # scenario gets its own XDG_RUNTIME_DIR, against which a bare + # "wayland-0" would resolve to nothing. + WAYLAND_DISPLAY="${WAYLAND_SOCKET}" export WAYLAND_DISPLAY echo "Wayland socket: ${WAYLAND_SOCKET}" export XDG_SESSION_TYPE=wayland GDK_BACKEND=wayland @@ -817,7 +816,7 @@ spec: PY=/home/bluefin-test/atspi-venv/bin/python # CHAIRLIFT_REQUIRE_ATSPI: a missing stack fails, it never skips. - if ! "${PY}" -c 'from dogtail.config import config; import behave, pyatspi, gi; gi.require_version("Gtk", "3.0"); gi.require_version("Gdk", "3.0"); import ponytail.ponytail'; then + if ! "${PY}" -c 'from dogtail.config import config; import behave, pyatspi, gi; gi.require_version("Gtk", "3.0"); gi.require_version("Gdk", "3.0"); import dogtail.hermetic.mutter'; then echo "accessibility stack is unavailable in the Wayland session" >&2 echo 5 >"${OUT}/behave-rc.txt" exit 5 @@ -844,7 +843,8 @@ spec: exit 0 fi BREW="$(cat "${G}/brew-path.txt")" - DATA=/home/bluefin-test/goose-isolation + # ChairLift's own profile location: $XDG_DATA_HOME, else ~/.local/share. + DATA="${HOME}/.local/share" PROFILE="${DATA}/chairlift/troubleshooting" # Every ChairLift step runs the way GNOME runs an app it launched: a # transient unit in the user manager's app.slice, inside the user's @@ -880,8 +880,6 @@ spec: fi SERVER="$(sed -n 's/^server=//p' "${G}/setup.log")" ls -la "${HOME}/.config" >"${G}/home-config-before.txt" 2>&1 - "${LANE}/bin/troubleshootcheck" write "${DATA}" "${SERVER}" >"${G}/profile-write.txt" 2>&1 || { echo "profile-write-failed" >"${G}/rc.txt"; exit 0; } - cp "${PROFILE}/goose/config/config.yaml" "${G}/config.before.yaml" # Agent Mode for real, through ChairLift's aistack on the stock host: # tap llmmanorg/tap, trust only the llmman formula, brew bundle, @@ -918,75 +916,108 @@ spec: exit 0 fi LLMMAN="$(sed -n 's/^llmman=//p' "${G}/agent-mode.log")" - MODEL_REPOS=(unsloth/Qwen3-0.6B-GGUF unsloth/Qwen3-1.7B-GGUF) - MRC=0 - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[0]}" >"${G}/model-1.log" 2>&1 "${G}/node.json" 2>&1 || true - "${LLMMAN}" ls >"${G}/llmman-ls.txt" 2>&1 || true - flatpak list --app --columns=application,installation >"${G}/flatpaks.txt" 2>&1 || true - if [ "${MRC}" -ne 0 ]; then - if ! grep -q 'not found in daemon stored models' "${G}/model-1.log"; then - echo "model-failed" >"${G}/rc.txt" - exit 0 - fi - # The pull itself completed; only aistack's stored-model check - # disagreed with llmman (model_pulled_and_alias_set stays false and - # fails the check). Set the alias through ChairLift's own - # ConfigureActiveModel so the launch and tool checks still run. - REF="$(sed -n 's/^ref=//p' "${G}/model-1.log")" - "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" alias "${REF}" >"${G}/alias-1.log" 2>&1 "${G}/rc.txt" - exit 0 - } + # Models through ChairLift's alias path: aistack.PullModel must + # succeed on its own (no lane fallback), then ConfigureActiveModel. + MODEL_REPOS=(unsloth/Qwen3-0.6B-GGUF unsloth/Qwen3-1.7B-GGUF unsloth/Qwen3-4B-GGUF) + model_step() { + local n="$1" rc=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[$((n - 1))]}" >"${G}/model-${n}.log" 2>&1 "${G}/model-${n}-rc.txt" + curl -s --max-time 5 http://127.0.0.1:17434/llmman/node >"${G}/node-${n}.json" 2>&1 || true + "${LLMMAN}" ls >"${G}/llmman-ls-${n}.txt" 2>&1 || true + return "${rc}" + } + if ! model_step 1; then + echo "model-failed" >"${G}/rc.txt" + exit 0 fi + flatpak list --app --columns=application,installation >"${G}/flatpaks.txt" 2>&1 || true "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 "${G}/launch.log" 2>&1 /dev/null | grep -q -- '--type=' || { GOOSE_MAIN="${pid}"; break; } - done + # ChairLift as a Homebrew install leaves it: the GUI and its wrapper + # in Homebrew's bin, where the distro's Ask Bluefin entry looks. + BIN="$(dirname "${BREW}")" + install -m 0755 "${LANE}/install/chairlift" "${BIN}/chairlift" + install -m 0755 "${LANE}/install/chairlift-wrapper" "${BIN}/chairlift-wrapper" + + # Launches run with GNOME Shell's own environment, the one a + # Custom Command Menu entry inherits, and a PATH without Homebrew, + # in a scope under app.slice, as GNOME starts an app. + SHELL_PID="$(pgrep -u "$(id -u)" -x gnome-shell | head -1)" + SHELL_ENV=() + while IFS= read -r -d '' kv; do + case "${kv}" in PATH=*) SHELL_PATH="${kv#PATH=}" ;; *) SHELL_ENV+=("${kv}") ;; esac + done < <(cat "/proc/${SHELL_PID}/environ" 2>/dev/null || sudo -n cat "/proc/${SHELL_PID}/environ") + MENU_PATH="$(printf '%s' "${SHELL_PATH:-/usr/bin:/bin}" | tr ':' '\n' | grep -v linuxbrew | paste -sd: -)" { - echo "alive=$( [ -n "${GOOSE_MAIN}" ] && kill -0 "${GOOSE_MAIN}" 2>/dev/null && echo yes || echo no)" - echo "goose_main_pid=${GOOSE_MAIN}" - echo "profile_lock=$(readlink "${PROFILE}/desktop/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" - echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" - } >"${G}/lock.txt" - if [ -n "${GOOSE_MAIN}" ]; then - # The Electron browser process's environ came back without any of - # these names, also through sudo (chairlift-wayland-lane-jhxkh, -t54w6); - # record what it does hold, and read the environment the app - # hands its own `goose serve` backend, which is what talks to the - # model. - sudo -n cat "/proc/${GOOSE_MAIN}/environ" | tr '\0' '\n' | cut -c1-120 | head -40 >"${G}/goose-main-environ-raw.txt" - GOOSED="$(pgrep -u "$(id -u)" -f '/usr/lib/Goose/resources/bin/goose serve' | head -1)" - echo "goosed_pid=${GOOSED}" >>"${G}/lock.txt" - [ -n "${GOOSED}" ] && tr '\0' '\n' <"/proc/${GOOSED}/environ" \ - | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|PATH|WAYLAND_DISPLAY|DISPLAY)=' \ - | grep -v -E '^GOOSE_SERVER__SECRET_KEY=' >"${G}/goose-environ.txt" - tr '\0' ' ' <"/proc/${GOOSE_MAIN}/cmdline" >"${G}/goose-cmdline.txt" - fi - ps -eo pid,args | grep -i -E 'goose|electron|llmman' | grep -v grep | cut -c1-400 >"${G}/processes.txt" || true - gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ - --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/goose-window.png" >"${G}/screenshot.txt" 2>&1 || true - pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null - sleep 5 - pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null - wait "${LAUNCH_PID}" 2>/dev/null + echo "gnome_shell_pid=${SHELL_PID}" + echo "shell_path=${SHELL_PATH:-}" + echo "menu_path=${MENU_PATH}" + printf '%s\n' "${SHELL_ENV[@]}" | grep -E '^(WAYLAND_DISPLAY|DISPLAY|XDG_RUNTIME_DIR|XDG_SESSION_TYPE|DBUS_SESSION_BUS_ADDRESS|HOME|XDG_DATA_HOME|XDG_CONFIG_HOME)=' + } >"${G}/shell-env.txt" + + # One launch, observed while Goose runs: its process tree, the + # environment llmman handed it (read from Goose's own `goose serve` + # backend: the Electron browser process's environ does not carry it), + # where its single-instance lock is, and a screenshot. + observe_launch() { + local tag="$1"; shift + systemd-run --user --scope --quiet --collect --slice=app.slice -p TasksMax=infinity \ + env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}" "$@" >"${G}/${tag}.log" 2>&1 /dev/null | grep -q -- '--type=' || { main="${pid}"; break; } + done + goosed="$(pgrep -u "$(id -u)" -f '/usr/lib/Goose/resources/bin/goose serve' | head -1)" + if kill -0 "${launcher}" 2>/dev/null; then rc=running; else wait "${launcher}"; rc=$?; fi + { + echo "launcher_exit=${rc}" + echo "alive=$( [ -n "${main}" ] && kill -0 "${main}" 2>/dev/null && echo yes || echo no)" + echo "goose_main_pid=${main}" + echo "goosed_pid=${goosed}" + echo "goose_parent=$( [ -n "${main}" ] && ps -o ppid= -p "${main}" | tr -d ' ')" + echo "profile_lock=$(readlink "${PROFILE}/desktop/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + } >"${G}/${tag}-lock.txt" + [ -n "${goosed}" ] && tr '\0' '\n' <"/proc/${goosed}/environ" \ + | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|WAYLAND_DISPLAY|DISPLAY)=' \ + | grep -v -E '^GOOSE_SERVER__SECRET_KEY=' >"${G}/${tag}-environ.txt" + [ -n "${main}" ] && tr '\0' ' ' <"/proc/${main}/cmdline" >"${G}/${tag}-cmdline.txt" + ps -eo pid,ppid,args | grep -i -E 'goose|electron|llmman|chairlift' | grep -v grep | cut -c1-400 >"${G}/${tag}-processes.txt" || true + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/${tag}.png" >"${G}/${tag}-screenshot.txt" 2>&1 || true + pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + sleep 5 + pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + if [ "${rc}" = running ]; then + pkill -TERM -u "$(id -u)" -x chairlift 2>/dev/null + wait "${launcher}" 2>/dev/null + fi + for _ in $(seq 1 20); do [ -L "${PROFILE}/desktop/Goose/SingletonLock" ] || break; sleep 0.5; done + } + # (3) A cold `chairlift --ask-bluefin`. + observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin + # (5) Exactly what the distro's Custom Command Menu entry runs. + observe_launch menu bash -c "${BIN}/chairlift-wrapper --ask-bluefin" # A real answer with tools: the goose CLI through llmman in the same - # profile, so the same extensions and hints. + # profile, so the same extensions and hints. The ground truth is + # what systemd itself reports as failed. + { + echo "system:"; systemctl --failed --no-legend --plain 2>&1 + echo "user:"; systemctl --user --failed --no-legend --plain 2>&1 + } >"${G}/failed-units-truth.txt" cat >"${G}/extract.py" <<'PYX' import json, os, re, sqlite3, sys - root, out = sys.argv[1], sys.argv[2] - calls, responses, seen = [], [], set() + root, out, seen_files = sys.argv[1], sys.argv[2], sys.argv[3:] + seen_ids = set() + for f in seen_files: + try: + seen_ids.update(c["id"] for c in json.load(open(f))["tool_calls"]) + except (OSError, ValueError, KeyError): + pass + calls, responses, seen, assistant_texts = [], [], set(), [] def walk(o): if isinstance(o, dict): kind = o.get("type") @@ -994,53 +1025,88 @@ spec: call = o.get("toolCall") or o.get("tool_call") or {} value = call.get("value", call) if isinstance(call, dict) else {} key = (o.get("id"), json.dumps(value, sort_keys=True, default=str)) - if key not in seen: + if key not in seen and o.get("id") not in seen_ids: seen.add(key) calls.append({"id": o.get("id"), "name": value.get("name"), "arguments": value.get("arguments")}) - if kind in ("toolResponse", "tool_response"): + if kind in ("toolResponse", "tool_response") and o.get("id") not in seen_ids: responses.append({"id": o.get("id"), "result": json.dumps(o.get("toolResult") or o.get("tool_result"), default=str)[:600]}) for v in o.values(): walk(v) elif isinstance(o, list): for v in o: walk(v) + def texts(o): + if isinstance(o, dict): + if o.get("type") == "text" and isinstance(o.get("text"), str): + yield o["text"] + for v in o.values(): + yield from texts(v) + elif isinstance(o, list): + for v in o: + yield from texts(v) db = os.path.join(root, "data", "sessions", "sessions.db") con = sqlite3.connect(db) tables = [r[0] for r in con.execute("select name from sqlite_master where type='table'")] for table in tables: - for row in con.execute(f'select * from "{table}"'): + cur = con.execute(f'select * from "{table}"') + columns = [d[0] for d in cur.description] + for row in cur: + record = dict(zip(columns, row)) for cell in row: if isinstance(cell, str) and re.search(r"tool_?[Rr]e(quest|sponse)", cell): try: walk(json.loads(cell)) except ValueError: pass + if table == "messages" and record.get("role") == "assistant": + for cell in row: + if isinstance(cell, str) and cell.startswith(("[", "{")): + try: + found = [t for t in texts(json.loads(cell)) if t.strip()] + except ValueError: + continue + if found: + assistant_texts.append("\n".join(found)) names = [c["name"] or "" for c in calls] json.dump({"tables": tables, "tool_calls": calls, "tool_responses": responses, + "final_text": assistant_texts[-1] if assistant_texts else "", "linux_tools_called": any(n.startswith("linux-tools__") for n in names), "search_knowledge_called": "bluefin-knowledge__search_knowledge" in names}, open(out, "w"), indent=1, default=str) PYX PROMPT="Use the linux-tools extension to list failed systemd units on this machine, then use search_knowledge to search the Project Bluefin knowledge base for 'bootc update'. Report both results." - for attempt in 1 2; do - if [ "${attempt}" -eq 2 ]; then - grep -q '"linux_tools_called": true' "${G}/tool-calls-1.json" 2>/dev/null \ - && grep -q '"search_knowledge_called": true' "${G}/tool-calls-1.json" 2>/dev/null && break - if ! "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[1]}" >"${G}/model-2.log" 2>&1 "${G}/alias-2.log" 2>&1 /dev/null; then + break fi + model_step "${n}" || break fi CRC=0 env GOOSE_PATH_ROOT="${PROFILE}/goose" XDG_CONFIG_HOME="${PROFILE}/desktop" \ - PATH="$(dirname "${BREW}"):/usr/bin:/bin" \ + PATH="${BIN}:/usr/bin:/bin" \ timeout 900 "${LLMMAN}" launch goose --model bluefin-active -- run --text "${PROMPT}" \ - >"${G}/cli-${attempt}.log" 2>&1 "${G}/cli-${attempt}-rc.txt" - python3 "${G}/extract.py" "${PROFILE}/goose" "${G}/tool-calls-${attempt}.json" >"${G}/extract-${attempt}.log" 2>&1 || true - latest="$(ls -t "${PROFILE}/goose/state/logs/cli"/*/* "${PROFILE}/goose/state/logs/cli"/* 2>/dev/null | grep -v '/$' | head -1)" - [ -n "${latest}" ] && [ -f "${latest}" ] && cp "${latest}" "${G}/cli-${attempt}-goose.log" + >"${G}/cli-${n}.log" 2>&1 "${G}/cli-${n}-rc.txt" + python3 "${G}/extract.py" "${PROFILE}/goose" "${G}/tool-calls-${n}.json" "${SEEN[@]}" >"${G}/extract-${n}.log" 2>&1 || true + SEEN+=("${G}/tool-calls-${n}.json") + python3 - "${G}/tool-calls-${n}.json" "${G}/failed-units-truth.txt" >"${G}/answer-${n}.txt" <<'PYQ' + import json, re, sys + result = json.load(open(sys.argv[1])) + truth = [line.split()[0] for line in open(sys.argv[2]).read().splitlines() + if line.strip() and not line.endswith(":") and not line.startswith(("system:", "user:"))] + text = result.get("final_text", "") + if truth: + correct = all(unit in text for unit in truth) + else: + correct = bool(re.search(r"\b(no|zero|0|none)\b[^.\n]{0,40}\bfail", text, re.I)) or "no failed" in text.lower() + print("quality=" + ("correct" if correct else "incorrect")) + print("truth=" + ",".join(truth)) + print("final_text=" + json.dumps(text)) + PYQ done cp "${PROFILE}/goose/config/config.yaml" "${G}/config.after.yaml" 2>/dev/null @@ -1125,19 +1191,52 @@ spec: tail -n 40 "${dir}chairlift.log" >&2 2>/dev/null || true done + # ---- walkthrough screenshots ------------------------------------ + # ChairLift's own capture (test/e2e/capture_walkthrough.sh) under its + # own wayland_session.sh: headless Mutter, native Wayland, frames from + # Mutter's ScreenCast, 900x700 as walkthrough_test.go sizes them. It + # runs on a private bus and runtime directory, beside the GDM session. + SHOTS_RC=0 + mapfile -t PAGE_NAMES < <(python3 -c 'import json; [print(i["name"]) for i in json.load(open("/workspace/lane/inventory/navigation.json"))]') + podman exec -i --user bluefin-test "${TARGET_NAME}" env -i \ + HOME=/home/bluefin-test PATH=/usr/bin:/bin LANG=C.UTF-8 \ + CHAIRLIFT_WAYLAND_SIZE=900x700 \ + CHAIRLIFT_SCHEMA_DIR=/home/bluefin-test/chairlift-schemas \ + CHAIRLIFT_ATSPI_PYTHON=/home/bluefin-test/atspi-venv/bin/python \ + bash -s -- "${PAGE_NAMES[@]}" <<'SHOTS' >/tmp/results/screenshots.log 2>&1 || SHOTS_RC=$? + set -euo pipefail + rm -rf /tmp/shots && mkdir -p /tmp/shots/bin /tmp/shots/runtime /tmp/results/screenshots-raw + chmod 0700 /tmp/shots/runtime + install -m 0755 /workspace/chairlift/build/e2e/chairlift /tmp/shots/bin/chairlift + export XDG_RUNTIME_DIR=/tmp/shots/runtime + timeout 600 dbus-run-session -- /workspace/chairlift/test/e2e/wayland_session.sh \ + /workspace/chairlift/test/e2e/capture_walkthrough.sh /tmp/shots/bin/chairlift /tmp/results/screenshots-raw "$@" + SHOTS + echo "${SHOTS_RC}" >/tmp/results/screenshots-rc.txt + # Only the PNGs are deliverable; the capture byproducts stay in -raw. + mkdir -p /tmp/results/screenshots + find /tmp/results/screenshots-raw -maxdepth 1 -name '[0-9]-*.png' -exec cp {} /tmp/results/screenshots/ \; + ls -l /tmp/results/screenshots >&2 + cp /workspace/lane/make-ci.log /workspace/lane/make-ci-rc /tmp/results/ 2>/dev/null || true + GOOSE_RC=0 G=/tmp/results/goose GOOSE_STATE=$(cat "${G}/rc.txt" 2>/dev/null || echo not-run) if [[ "${GOOSE_STATE}" == "skipped" ]]; then echo '{"status":"skipped"}' >"${G}/verdict.json" else - echo "=== Goose isolation (${GOOSE_STATE}) ===" >&2 - for f in setup.log setup-rc.txt brew-installed.txt tap-info.json tap-info.err trust-state.txt polkit-session.txt \ - agent-mode.log agent-mode-rc.txt llmman-unit.txt flatpaks.txt node.json llmman-ls.txt alias-1.log alias-2.log \ - model-1.log model-2.log command-env.txt launch.log lock.txt goose-environ.txt goose-main-environ-raw.txt goose-cmdline.txt \ - cli-1-rc.txt cli-1.log tool-calls-1.json cli-2-rc.txt cli-2.log tool-calls-2.json; do + echo "=== Engine (${GOOSE_STATE}) ===" >&2 + for f in setup.log setup-rc.txt brew-installed.txt tap-info.json trust-state.txt polkit-session.txt \ + agent-mode.log agent-mode-rc.txt flatpaks.txt model-1.log model-1-rc.txt node-1.json llmman-ls-1.txt \ + command-env.txt shell-env.txt \ + askbluefin.log askbluefin-lock.txt askbluefin-environ.txt askbluefin-cmdline.txt askbluefin-processes.txt \ + menu.log menu-lock.txt menu-environ.txt menu-cmdline.txt menu-processes.txt \ + failed-units-truth.txt \ + model-2.log model-2-rc.txt model-3.log model-3-rc.txt \ + cli-1-rc.txt cli-1.log answer-1.txt cli-2-rc.txt cli-2.log answer-2.txt cli-3-rc.txt cli-3.log answer-3.txt \ + enabled-extensions.txt home-goose-paths.txt; do echo "--- ${f}" >&2 - cat "${G}/${f}" >&2 2>/dev/null || true + cut -c1-600 "${G}/${f}" >&2 2>/dev/null || true done GOOSE_STATE="${GOOSE_STATE}" python3 - <<'PY' || GOOSE_RC=$? import json, os, re, sys @@ -1147,73 +1246,95 @@ spec: return open(os.path.join(g, name), encoding="utf-8", errors="replace").read() except OSError: return "" + def kv(name): + return dict(line.split("=", 1) for line in read(name).splitlines() if "=" in line) state = os.environ["GOOSE_STATE"] - lock = dict(line.split("=", 1) for line in read("lock.txt").split() if "=" in line) enabled = sorted(read("enabled-extensions.txt").split()) paths = read("home-goose-paths.txt") tree = read("profile-tree.txt").splitlines() - root = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting/goose" - def kv(name): - return dict(line.split("=", 1) for line in read(name).splitlines() if "=" in line) + profile = "/home/bluefin-test/.local/share/chairlift/troubleshooting" setup = kv("setup.log") command_path = kv("command-env.txt").get("PATH", "") brew_bin = os.path.dirname(read("brew-path.txt").strip()) agent = kv("agent-mode.log") - model = kv("model-2.log") if read("model-2.log") else kv("model-1.log") - desktop_model = kv("model-1.log") - environ = kv("goose-environ.txt") - attempt = "2" if read("cli-2-rc.txt") else "1" - try: - tools = json.loads(read(f"tool-calls-{attempt}.json")) - except ValueError: - tools = {} + model = kv("model-1.log") try: tap_info = json.loads(read("tap-info.json")) tap_trusted = tap_info[0].get("trusted") if tap_info else None except (ValueError, IndexError, AttributeError): tap_trusted = None - profile = "/home/bluefin-test/goose-isolation/chairlift/troubleshooting" + try: + tools = json.loads(read("tool-calls-1.json")) + except ValueError: + tools = {} + + def launch_checks(tag): + lock = kv(f"{tag}-lock.txt") + env = kv(f"{tag}-environ.txt") + log = read(f"{tag}.log") + return { + f"{tag}_launcher_exit_zero": lock.get("launcher_exit") == "0", + f"{tag}_goose_running": lock.get("alive") == "yes", + f"{tag}_no_failure_logged": not re.search( + r"app: (launch goose desktop failed|goose desktop exited with error)|^Error:|^\[llmman\].*refus", + log, re.M), + f"{tag}_env_goose_path_root": env.get("GOOSE_PATH_ROOT") == profile + "/goose", + f"{tag}_env_xdg_config_home": env.get("XDG_CONFIG_HOME") == profile + "/desktop", + f"{tag}_env_goose_provider_openai": env.get("GOOSE_PROVIDER") == "openai", + f"{tag}_env_openai_host_agent_mode": "127.0.0.1:17434" in env.get("OPENAI_HOST", ""), + f"{tag}_env_goose_model_set": bool(env.get("GOOSE_MODEL")), + f"{tag}_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", + f"{tag}_no_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", + }, lock, env + + ask_checks, ask_lock, ask_env = launch_checks("askbluefin") + menu_checks, menu_lock, menu_env = launch_checks("menu") checks = { "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", - "isolation_run_completed": state == "done", "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), "agent_mode_enabled_on_stock_host": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", "llmman_formula_trusted": "llmmanorg/tap/llmman" in read("trust-state.txt"), "llmmanorg_tap_not_trusted": tap_trusted is False, - "model_pulled_and_alias_set": bool(desktop_model.get("ref")) and desktop_model.get("active") == desktop_model.get("ref"), - "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin - and command_path.split(":")[1:] == ["/usr/bin", "/bin"], - "llmman_launch_not_refused": lock.get("alive") == "yes" - and not re.search(r"^(Error|\[llmman\].*refus)", read("launch.log"), re.M), - "environ_goose_path_root": environ.get("GOOSE_PATH_ROOT") == profile + "/goose", - "environ_xdg_config_home": environ.get("XDG_CONFIG_HOME") == profile + "/desktop", - "environ_goose_provider_openai": environ.get("GOOSE_PROVIDER") == "openai", - "environ_openai_host_agent_mode": "127.0.0.1:17434" in environ.get("OPENAI_HOST", ""), - "environ_goose_model_is_resolved_ref": environ.get("GOOSE_MODEL") == desktop_model.get("ref"), - "singleton_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", - "no_singleton_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", - "cli_exit_zero": read(f"cli-{attempt}-rc.txt").strip() == "0", + "pullmodel_succeeded_and_alias_set": read("model-1-rc.txt").strip() == "0" + and bool(model.get("ref")) and model.get("active") == model.get("ref"), + "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin, + **ask_checks, + **menu_checks, + "menu_same_goose_env_as_ask_bluefin": bool(ask_env) and all( + ask_env.get(k) == menu_env.get(k) + for k in ("GOOSE_PATH_ROOT", "XDG_CONFIG_HOME", "GOOSE_PROVIDER", "OPENAI_HOST", "GOOSE_MODEL")), + "cli_exit_zero": read("cli-1-rc.txt").strip() == "0", "linux_tools_tool_called": tools.get("linux_tools_called") is True, "search_knowledge_tool_called": tools.get("search_knowledge_called") is True, "only_linux_tools_and_bluefin_knowledge_enabled": enabled == ["bluefin-knowledge", "linux-tools"], - "goose_data_or_state_under_path_root": any(p.startswith(root + "/data/") or p.startswith(root + "/state/") for p in tree), + "goose_data_or_state_under_path_root": any(p.startswith(profile + "/goose/data/") or p.startswith(profile + "/goose/state/") for p in tree), "nothing_in_home_config_goose": "ABSENT /home/bluefin-test/.config/goose" in paths, } - verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, - "checks": checks, "lock": lock, "enabled": enabled, "cli_attempt": attempt, - "model": model, "desktop_model": desktop_model, "environ": environ, - "tool_calls": [c.get("name") for c in tools.get("tool_calls", [])]} + answers = {} + for n in ("1", "2", "3"): + if read(f"cli-{n}-rc.txt"): + answers[n] = {"model": kv(f"model-{n}.log").get("ref") if n != "1" else model.get("ref"), + "pull": {k: kv(f"model-{n}.log").get(k) for k in ("size_bytes", "pull_seconds")}, + "cli_rc": read(f"cli-{n}-rc.txt").strip(), + "quality": kv(f"answer-{n}.txt").get("quality"), + "tool_calls": [c.get("name") for c in json.loads(read(f"tool-calls-{n}.json") or "{}").get("tool_calls", [])]} + verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, "checks": checks, + "model": model, "askbluefin": {"lock": ask_lock, "env": ask_env}, + "menu": {"lock": menu_lock, "env": menu_env}, "enabled": enabled, "answers": answers} json.dump(verdict, open(os.path.join(g, "verdict.json"), "w"), indent=1) print(json.dumps(verdict, indent=1)) - print(paths) sys.exit(0 if verdict["status"] == "pass" else 1) PY - tail -n 60 "${G}/launch.log" >&2 2>/dev/null || true fi - echo "behave exit ${BEHAVE_RC}; suite verdict ${SUITE_RC}; goose verdict ${GOOSE_RC}" >&2 - if [[ "${BEHAVE_RC}" != "0" || "${SUITE_RC}" != "0" || "${GOOSE_RC}" != "0" ]]; then + CI_RC=$(cat /workspace/lane/make-ci-rc 2>/dev/null || echo missing) + SHOTS_OK=0 + { [ "${SHOTS_RC}" = 0 ] && grep -q 'walkthrough complete' /tmp/results/screenshots.log \ + && grep -q 'views: reset group built' /tmp/results/screenshots-raw/chairlift.log \ + && [ "$(find /tmp/results/screenshots -name '*.png' | wc -l)" -eq $(( ${#PAGE_NAMES[@]} + 1 )) ]; } || SHOTS_OK=1 + echo "make ci exit ${CI_RC}; behave exit ${BEHAVE_RC}; suite verdict ${SUITE_RC}; engine verdict ${GOOSE_RC}; screenshots ${SHOTS_OK}" | tee /tmp/results/lane-summary.txt >&2 + if [[ "${CI_RC}" != "0" || "${BEHAVE_RC}" != "0" || "${SUITE_RC}" != "0" || "${GOOSE_RC}" != "0" || "${SHOTS_OK}" != "0" ]]; then exit 1 fi volumes: diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index cc48620c16..f498e4697d 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -53,7 +53,7 @@ def test_runs_on_ghost_behind_the_container_qa_semaphore(): assert template["nodeSelector"]["kubernetes.io/hostname"] == "ghost" keys = [s["configMapKeyRef"]["key"] for s in template["synchronization"]["semaphores"]] assert keys == ["ghost-container-qa"] - assert template["activeDeadlineSeconds"] <= 5400 + assert template["activeDeadlineSeconds"] <= 7200 def test_builds_the_requested_branch_with_the_e2e_target(): @@ -61,11 +61,16 @@ def test_builds_the_requested_branch_with_the_e2e_target(): script = build["args"][0] assert 'git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}"' in script assert "CGO_ENABLED=0 make build-e2e schemas" in script + # ChairLift's whole CI gate runs on the untouched checkout, with the + # golangci-lint CI pins, verified by checksum, before any lane tool exists. + assert script.index("make ci >") < script.index("build/lane/navjson") + assert 'echo "${LINT_SHA256} /tmp/${LINT_TGZ}" | sha256sum -c -' in script assert "troubleshoot.ProfileAt(os.Args[2])" in script # The install goes through ChairLift's own Setup, not hand-written brew. assert "troubleshoot.Setup(troubleshoot.Detect()," in script - assert "troubleshoot.Command(state," in script assert "navigation.Items()" in script + # Installed the way the Homebrew cask lays ChairLift out. + assert "install -m 0755 data/chairlift-wrapper.sh /workspace/lane/install/chairlift-wrapper" in script _bash_parses(script) @@ -127,7 +132,7 @@ def test_fails_closed_and_cleans_up(): assert "sys.exit(1)" in source -def test_goose_check_asserts_profile_isolation(): +def test_engine_check_runs_chairlifts_own_paths(): runner = _heredoc(_runner(), "RUNNER") # ChairLift installs the packages itself, with no Homebrew on PATH and # no `brew shellenv`, the way a direct launch runs it. @@ -136,42 +141,57 @@ def test_goose_check_asserts_profile_isolation(): assert '"${LANE}/bin/troubleshootcheck" command-path' in runner assert 'eval "$("${BREW}" shellenv)"' not in runner assert '"${BREW}" install' not in runner - # Agent Mode, the model, and the launch are ChairLift's own code paths. - for step in ("agent-mode", 'model "${MODEL_REPOS[0]}"', 'launch "${DATA}"'): - assert f'"${{LANE}}/bin/troubleshootcheck" {step}' in runner, step - assert "/usr/bin/true" not in _template()["initContainers"][0]["args"][0] + assert '"${LANE}/bin/troubleshootcheck" agent-mode' in runner + assert '"${LANE}/bin/troubleshootcheck" model' in runner + # No lane fallbacks: PullModel must succeed, and nothing is trusted or + # aliased on ChairLift's behalf. + for fallback in ("troubleshootcheck alias", "not found in daemon stored models", '"${BREW}" trust', "brew trust"): + assert fallback not in runner, fallback + build = _template()["initContainers"][0]["args"][0] + assert "/usr/bin/true" not in build + for call in ("aistack.Enable(ctx)", "aistack.WaitHealthy(", "aistack.PullModel(", "aistack.ConfigureActiveModel("): + assert call in build, call + # Goose is launched by ChairLift itself: a cold --ask-bluefin, and the + # distro menu's exact command, both with GNOME Shell's environment. + assert 'observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin' in runner + assert 'observe_launch menu bash -c "${BIN}/chairlift-wrapper --ask-bluefin"' in runner + assert 'env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}"' in runner + assert "grep -v linuxbrew" in runner # Nothing fakes the provider: llmman hands Goose its environment. for fake in ("GOOSE_PROVIDER=openai", "OPENAI_HOST=http://127.0.0.1:9", "OPENAI_API_KEY=x"): assert fake not in runner, fake assert 'launch goose --model bluefin-active -- run --text' in runner - # Agent Mode must work on the stock host: the lane trusts nothing itself. - assert '"${BREW}" trust' not in runner and "brew trust" not in runner - build = _template()["initContainers"][0]["args"][0] - for call in ("aistack.Enable(ctx)", "aistack.WaitHealthy(", "aistack.PullModel(", "aistack.ConfigureActiveModel(", - "troubleshoot.Command(state, profile, aistack.ActiveModelAlias)"): - assert call in build, call + assert "systemctl --failed --no-legend --plain" in runner source = _runner() for check in ( "setup_ran_without_brew_on_path", "chairlift_setup_exit_zero", "goose_desktop_resolves_after_setup", - "command_env_path_has_brew_bin_first", "agent_mode_enabled_on_stock_host", "llmman_formula_trusted", "llmmanorg_tap_not_trusted", - "model_pulled_and_alias_set", - "llmman_launch_not_refused", - "environ_goose_path_root", - "environ_xdg_config_home", - "environ_goose_provider_openai", - "environ_openai_host_agent_mode", - "environ_goose_model_is_resolved_ref", + "pullmodel_succeeded_and_alias_set", + "command_env_path_has_brew_bin_first", + "_launcher_exit_zero", + "_env_goose_path_root", + "_env_openai_host_agent_mode", + "_lock_in_profile", + "menu_same_goose_env_as_ask_bluefin", "linux_tools_tool_called", "search_knowledge_tool_called", - "singleton_lock_in_profile", - "no_singleton_lock_in_home_config", "only_linux_tools_and_bluefin_knowledge_enabled", "goose_data_or_state_under_path_root", "nothing_in_home_config_goose", ): assert check in source, check + + +def test_screenshots_use_chairlifts_own_wayland_capture(): + source = _runner() + assert "/workspace/chairlift/test/e2e/wayland_session.sh" in source + assert "/workspace/chairlift/test/e2e/capture_walkthrough.sh" in source + assert "CHAIRLIFT_WAYLAND_SIZE=900x700" in source + assert "views: reset group built" in source + # Byproducts stay behind; only the page PNGs are deliverable. + assert "-name '[0-9]-*.png'" in source + assert 'CI_RC}" != "0"' in source From f45debfe0bd10b3ed55ad6a78ff67b204e89d4a7 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 17:42:59 -0400 Subject: [PATCH 15/19] fix(argo): accept llmman's registry-qualified alias value ReadActiveModel answers with the name llmman stored the model under, hf.co/ (chairlift-wayland-lane-r5vxf), after a PullModel that succeeded on its own. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- argo/workflow-templates/run-chairlift-wayland-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 762ee0ac96..365648e613 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -295,7 +295,9 @@ spec: fail(err) } fmt.Println("active=" + active) - if active != ref { + // llmman answers with the registry-qualified name it stored the + // model under (hf.co/), observed in chairlift-wayland-lane-r5vxf. + if active != ref && !strings.HasSuffix(active, "/"+ref) { fail(fmt.Errorf("alias %s reads %q, want %q", aistack.ActiveModelAlias, active, ref)) } case len(os.Args) == 3 && os.Args[1] == "command-path": @@ -1297,7 +1299,7 @@ spec: "llmman_formula_trusted": "llmmanorg/tap/llmman" in read("trust-state.txt"), "llmmanorg_tap_not_trusted": tap_trusted is False, "pullmodel_succeeded_and_alias_set": read("model-1-rc.txt").strip() == "0" - and bool(model.get("ref")) and model.get("active") == model.get("ref"), + and bool(model.get("ref")) and model.get("active", "") in (model["ref"], "hf.co/" + model["ref"]), "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin, **ask_checks, **menu_checks, From 2c46aa0b186827784bb667b3ffd0b3b56bd5c3e1 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 18:09:43 -0400 Subject: [PATCH 16/19] docs(workflow-reference): describe the final ChairLift lane make ci gate, ask-bluefin and menu-command launches, PullModel without a fallback, answer-quality recording, and ChairLift's own Wayland capture. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/reference/workflow-reference.md | 70 ++++++++++++++++++---------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index 54dd4c3174..afa3ad30f1 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -135,18 +135,28 @@ before retrying. (`ghcr.io/projectbluefin/dakota`), `image-tag` (`testing`), `behave-tags` (empty = every scenario), `goose-isolation-check` (`true`). - **Shape:** a `golang:1.26` init container clones the branch and runs - `make build-e2e schemas`, plus two lane tools built inside the module: - `navjson` (the page/shortcut inventory the Go gate normally passes) and - `troubleshootcheck` (drives the branch's `internal/troubleshoot`: - `Setup(Detect(), …)`, the PATH `Command` hands the launch, - `ProfileAt(dir).Write`, and a config reader). - The runner then boots `run-container-tests`' nested systemd/GDM target — - headless GNOME Shell drop-in, linger, `lastchg`, resolver bind — with - `/usr/share/chairlift` masked and `/proc/cmdline` blanked like - `dakota_atspi.sh`, and runs behave from a hash-pinned venv under - `qecore-headless --session-type wayland`, with `GDK_BACKEND=wayland` and - `DISPLAY` unset, so neither the app nor Goose can fall back to the - session's on-demand Xwayland. Xvfb is never used. + ChairLift's `make ci` on the untouched checkout. It fetches golangci-lint + v2.12.2, the version CI pins, by checksum, and records the result, which + the verdict gates on. It then runs `make build-e2e schemas`, stages + `chairlift` and `chairlift-wrapper` the way the Homebrew cask installs + them, and builds two lane tools inside the module: `navjson` (the + page/shortcut inventory the Go gate normally passes) and + `troubleshootcheck`. `troubleshootcheck` drives the branch's + `internal/troubleshoot` and `internal/aistack`: Setup, Agent Mode, the + model alias path, the PATH `Command` hands the launch, and a config + reader. + The runner then boots `run-container-tests`' nested systemd/GDM target: + headless GNOME Shell drop-in, linger, `lastchg`, and the resolver bind. + `/usr/share/chairlift` is masked and `/proc/cmdline` blanked, as the fixture + harness does. Behave runs from a hash-pinned venv under + `qecore-headless --session-type wayland`, with `GDK_BACKEND=wayland`, an + absolute `WAYLAND_DISPLAY`, and `DISPLAY` unset. Keys go through dogtail's + Mutter RemoteDesktop backend. Xvfb is never used. +- **Screenshots:** ChairLift's own `capture_walkthrough.sh` runs under its + `wayland_session.sh` (headless Mutter at 900x700, frames from Mutter's + ScreenCast) inside the target, on a private bus beside the GDM session. + Only the page PNGs are kept, in `screenshots/`; the byproducts stay in + `screenshots-raw/`. - **Wayland traps:** qecore hands the script gnome-session's environment, which has no `WAYLAND_DISPLAY`; the script reads it from `systemctl --user show-environment`. Mutter's focus-stealing prevention @@ -163,19 +173,25 @@ before retrying. 2. `aistack.Enable` turns Agent Mode on, on the stock host. The check records that `brew trust` covers the `llmman` formula and not `llmmanorg/tap`. - 3. The smallest Qwen3 is resolved through aistack's live resolver, pulled, - and set as the `bluefin-active` alias. The lane also records - `/llmman/node` and `llmman ls`. - 4. The real `troubleshoot.Command` launch - (`llmman launch goose-desktop`) runs in the Wayland session. The check - reads the Goose process's environment (`GOOSE_PATH_ROOT`, + 3. The smallest Qwen3 is resolved through aistack's live resolver, pulled + with `aistack.PullModel` (which must succeed on its own), and set as the + `bluefin-active` alias. llmman reports the alias as the + registry-qualified `hf.co/`. The lane also records `/llmman/node` + and `llmman ls`. + 4. Two launches run with GNOME Shell's own environment and a PATH without + Homebrew, in an app.slice scope. The first is a cold + `chairlift --ask-bluefin`. The second is the distro menu's exact + command, `bash -c '…/chairlift-wrapper --ask-bluefin'`. Each must + exit 0 with Goose running through llmman. The lane reads the + environment Goose hands its `goose serve` backend (`GOOSE_PATH_ROOT`, `XDG_CONFIG_HOME`, `GOOSE_PROVIDER=openai`, `OPENAI_HOST` on - `127.0.0.1:17434`, `GOOSE_MODEL`), checks where its lock is, and takes - a screenshot. + `127.0.0.1:17434`, `GOOSE_MODEL`); the two launches must agree on it. + It also checks that the lock is in the profile and takes a screenshot. 5. `llmman launch goose -- run --text …`, run in the same profile, must call a `linux-tools__*` tool and `bluefin-knowledge__search_knowledge`, - read back from `data/sessions/sessions.db`. If the 0.6B model will not - call tools, the lane retries once with the next size up. + read back from `data/sessions/sessions.db`. The answer is compared + with `systemctl --failed` for 0.6B, 1.7B, and then 4B, and recorded. + Answer quality does not gate the run. Throughout, the profile's lock, data, state, and config must stay inside it, and only `linux-tools` and `bluefin-knowledge` may be enabled. @@ -187,8 +203,14 @@ before retrying. user joins `wheel`, as Bluefin's first user does. `TasksMax` is lifted on the user slice and on the units, because brew's download threads hit the container-sized default. -- **Verdicts:** fails when behave fails, when 0 scenarios executed, or when - any Goose check fails. Outputs `result`, `failed-scenarios`, and +- **Verdicts:** the run fails if any of these happens: + - `make ci` fails; + - behave fails, or executes 0 scenarios; + - any engine check fails; + - the screenshot capture does not produce every page and the + reset-group marker. + + Outputs `result`, `failed-scenarios`, and `goose-isolation` (JSON). Everything under `/tmp/results` (behave log, JUnit, per-scenario `chairlift.log`, `tree.txt`, `screen.png`; Goose `setup.log`, `command-env.txt`, lock evidence, config, screenshot) is persisted to From c55d8c9a5a0441a94992725e725f875999923585 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 19:16:06 -0400 Subject: [PATCH 17/19] feat(argo): check a second Ask Bluefin while Goose is running; optional answer-quality runs With a Goose session from a cold --ask-bluefin still running (its window minimized), run the distro menu command again and assert exit 0, the same single Goose main process, and a Goose window afterwards; record whether it came back unminimized and focused. answer-quality=false keeps only the smallest model's tool-call check. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 75 +++++++++++++++++-- .../unit/test_run_chairlift_wayland_tests.py | 11 +++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 365648e613..f3686272ec 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -39,6 +39,9 @@ spec: value: "" - name: goose-isolation-check value: "true" + # false: only the smallest model's tool-call check, no larger models. + - name: answer-quality + value: "true" templates: - name: run-chairlift-wayland-tests # Same placement as run-container-tests: ghost is the only node with a @@ -72,6 +75,8 @@ spec: value: "{{workflow.parameters.behave-tags}}" - name: goose-isolation-check value: "{{workflow.parameters.goose-isolation-check}}" + - name: answer-quality + value: "{{workflow.parameters.answer-quality}}" outputs: parameters: - name: result @@ -378,6 +383,8 @@ spec: value: "{{inputs.parameters.behave-tags}}" - name: GOOSE_CHECK value: "{{inputs.parameters.goose-isolation-check}}" + - name: ANSWER_QUALITY + value: "{{inputs.parameters.answer-quality}}" volumeMounts: - mountPath: /workspace name: workspace @@ -737,6 +744,7 @@ spec: # qecore-headless replaces its environment with the session's. printf '%s' "${BEHAVE_TAGS}" >/workspace/lane/behave-tags printf '%s' "${GOOSE_CHECK}" >/workspace/lane/goose-check + printf '%s' "${ANSWER_QUALITY}" >/workspace/lane/answer-quality cat >/workspace/lane/run-chairlift.sh <<'RUNNER' #!/bin/bash # Runs inside the bluefin-test GNOME Wayland session (qecore-headless). @@ -989,17 +997,52 @@ spec: ps -eo pid,ppid,args | grep -i -E 'goose|electron|llmman|chairlift' | grep -v grep | cut -c1-400 >"${G}/${tag}-processes.txt" || true gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/${tag}.png" >"${G}/${tag}-screenshot.txt" 2>&1 || true - pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null - sleep 5 - pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + [ "${KEEP_GOOSE:-0}" = 1 ] || stop_goose if [ "${rc}" = running ]; then pkill -TERM -u "$(id -u)" -x chairlift 2>/dev/null wait "${launcher}" 2>/dev/null fi + } + stop_goose() { + pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + sleep 5 + pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null for _ in $(seq 1 20); do [ -L "${PROFILE}/desktop/Goose/SingletonLock" ] || break; sleep 0.5; done } - # (3) A cold `chairlift --ask-bluefin`. - observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin + goose_mains() { + local pid + for pid in $(pgrep -u "$(id -u)" -f '/usr/lib/Goose/Goose'); do + tr '\0' ' ' <"/proc/${pid}/cmdline" 2>/dev/null | grep -q -- '--type=' || echo "${pid}" + done + } + goose_windows() { + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell --method org.gnome.Shell.Eval \ + "let ws = global.get_window_actors().map(a => a.meta_window).filter(w => /goose/i.test(w.get_wm_class() || '')); let f = global.display.focus_window; JSON.stringify({windows: ws.length, minimized: ws.filter(w => w.minimized).length, focused: !!f && /goose/i.test(f.get_wm_class() || '')})" 2>&1 + } + # (3) A cold `chairlift --ask-bluefin`; Goose is left running. + KEEP_GOOSE=1 observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin + # (6) The menu again while that session runs: Goose's single-instance + # lock must bring the running window forward rather than start a + # second Goose. The window is minimized first, so "brought forward" + # is visible as unminimized and focused. + { + echo "mains_before=$(goose_mains | paste -sd, -)" + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell --method org.gnome.Shell.Eval \ + "global.get_window_actors().map(a => a.meta_window).filter(w => /goose/i.test(w.get_wm_class() || '')).forEach(w => w.minimize()); true" >/dev/null 2>&1 + sleep 1 + echo "windows_before=$(goose_windows)" + RRC=0 + timeout 60 systemd-run --user --scope --quiet --collect --slice=app.slice -p TasksMax=infinity \ + env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}" bash -c "${BIN}/chairlift-wrapper --ask-bluefin" \ + >"${G}/relaunch.log" 2>&1 "${G}/relaunch-state.txt" 2>&1 + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/relaunch.png" >/dev/null 2>&1 || true + stop_goose # (5) Exactly what the distro's Custom Command Menu entry runs. observe_launch menu bash -c "${BIN}/chairlift-wrapper --ask-bluefin" @@ -1080,6 +1123,7 @@ spec: SEEN=() for n in 1 2 3; do if [ "${n}" -gt 1 ]; then + [ "$(cat "${LANE}/answer-quality")" = "true" ] || break # Answer quality: stop at the first model after the smallest # that names every failed unit systemd reports. if [ "${n}" -gt 2 ] && grep -q '^quality=correct' "${G}/answer-$((n - 1)).txt" 2>/dev/null; then @@ -1232,7 +1276,7 @@ spec: agent-mode.log agent-mode-rc.txt flatpaks.txt model-1.log model-1-rc.txt node-1.json llmman-ls-1.txt \ command-env.txt shell-env.txt \ askbluefin.log askbluefin-lock.txt askbluefin-environ.txt askbluefin-cmdline.txt askbluefin-processes.txt \ - menu.log menu-lock.txt menu-environ.txt menu-cmdline.txt menu-processes.txt \ + menu.log menu-lock.txt menu-environ.txt menu-cmdline.txt menu-processes.txt relaunch-state.txt relaunch.log \ failed-units-truth.txt \ model-2.log model-2-rc.txt model-3.log model-3-rc.txt \ cli-1-rc.txt cli-1.log answer-1.txt cli-2-rc.txt cli-2.log answer-2.txt cli-3-rc.txt cli-3.log answer-3.txt \ @@ -1290,6 +1334,20 @@ spec: }, lock, env ask_checks, ask_lock, ask_env = launch_checks("askbluefin") + relaunch = kv("relaunch-state.txt") + def shell_json(value): + match = re.search(r"'(\{.*\})'", value or "") + try: + return json.loads(match.group(1)) if match else {} + except ValueError: + return {} + windows_after = shell_json(relaunch.get("windows_after")) + relaunch_checks = { + "relaunch_while_running_exit_zero": relaunch.get("relaunch_exit") == "0", + "relaunch_no_second_goose": bool(relaunch.get("mains_before")) + and relaunch.get("mains_after") == relaunch.get("mains_before"), + "relaunch_goose_window_exists": windows_after.get("windows", 0) >= 1, + } menu_checks, menu_lock, menu_env = launch_checks("menu") checks = { "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", @@ -1303,6 +1361,7 @@ spec: "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin, **ask_checks, **menu_checks, + **relaunch_checks, "menu_same_goose_env_as_ask_bluefin": bool(ask_env) and all( ask_env.get(k) == menu_env.get(k) for k in ("GOOSE_PATH_ROOT", "XDG_CONFIG_HOME", "GOOSE_PROVIDER", "OPENAI_HOST", "GOOSE_MODEL")), @@ -1323,7 +1382,9 @@ spec: "tool_calls": [c.get("name") for c in json.loads(read(f"tool-calls-{n}.json") or "{}").get("tool_calls", [])]} verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, "checks": checks, "model": model, "askbluefin": {"lock": ask_lock, "env": ask_env}, - "menu": {"lock": menu_lock, "env": menu_env}, "enabled": enabled, "answers": answers} + "menu": {"lock": menu_lock, "env": menu_env}, "enabled": enabled, "answers": answers, + "relaunch": {"state": relaunch, "windows_before": shell_json(relaunch.get("windows_before")), + "windows_after": windows_after}} json.dump(verdict, open(os.path.join(g, "verdict.json"), "w"), indent=1) print(json.dumps(verdict, indent=1)) sys.exit(0 if verdict["status"] == "pass" else 1) diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index f498e4697d..3a720e22d6 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -42,6 +42,7 @@ def test_parameters_are_the_lane_contract(): "image-tag", "behave-tags", "goose-isolation-check", + "answer-quality", ] defaults = {p["name"]: p["value"] for p in _doc()["spec"]["arguments"]["parameters"]} assert defaults["image"] == "ghcr.io/projectbluefin/dakota" @@ -195,3 +196,13 @@ def test_screenshots_use_chairlifts_own_wayland_capture(): # Byproducts stay behind; only the page PNGs are deliverable. assert "-name '[0-9]-*.png'" in source assert 'CI_RC}" != "0"' in source + + +def test_relaunch_while_goose_runs_reuses_the_session(): + runner = _heredoc(_runner(), "RUNNER") + assert 'KEEP_GOOSE=1 observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin' in runner + assert 'bash -c "${BIN}/chairlift-wrapper --ask-bluefin"' in runner + assert "mains_before=" in runner and "mains_after=" in runner + source = _runner() + for check in ("relaunch_while_running_exit_zero", "relaunch_no_second_goose", "relaunch_goose_window_exists"): + assert check in source, check From 9b4ce97282f62f329f069662207098c9053365e1 Mon Sep 17 00:00:00 2001 From: pi Date: Mon, 5 Oct 2026 19:39:03 -0400 Subject: [PATCH 18/19] fix(argo): parse Shell.Eval's doubly encoded JSON; check the window is raised Shell.Eval answers a JSON.stringify result as a JS string inside a GVariant string, so the relaunch window state never parsed and relaunch_goose_window_exists read false although one Goose window existed (chairlift-wayland-lane-n7fsg). Also assert the existing window comes back unminimized and focused, which that run shows it did not. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../run-chairlift-wayland-tests.yaml | 10 ++++++++-- tests/unit/test_run_chairlift_wayland_tests.py | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index f3686272ec..3064f4cbac 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -1336,9 +1336,14 @@ spec: ask_checks, ask_lock, ask_env = launch_checks("askbluefin") relaunch = kv("relaunch-state.txt") def shell_json(value): - match = re.search(r"'(\{.*\})'", value or "") + # Shell.Eval returns the stringified result as a JS string, inside + # a GVariant string: (true, '"{\\"windows\\":1,…}"'). + match = re.search(r"\(true, '(.*)'\)\s*$", value or "") + if not match: + return {} try: - return json.loads(match.group(1)) if match else {} + decoded = json.loads(match.group(1).replace("\\\\", "\\")) + return json.loads(decoded) if isinstance(decoded, str) else decoded except ValueError: return {} windows_after = shell_json(relaunch.get("windows_after")) @@ -1347,6 +1352,7 @@ spec: "relaunch_no_second_goose": bool(relaunch.get("mains_before")) and relaunch.get("mains_after") == relaunch.get("mains_before"), "relaunch_goose_window_exists": windows_after.get("windows", 0) >= 1, + "relaunch_goose_window_raised": windows_after.get("minimized", 1) == 0 and windows_after.get("focused") is True, } menu_checks, menu_lock, menu_env = launch_checks("menu") checks = { diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py index 3a720e22d6..ebf79936ea 100644 --- a/tests/unit/test_run_chairlift_wayland_tests.py +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -204,5 +204,6 @@ def test_relaunch_while_goose_runs_reuses_the_session(): assert 'bash -c "${BIN}/chairlift-wrapper --ask-bluefin"' in runner assert "mains_before=" in runner and "mains_after=" in runner source = _runner() - for check in ("relaunch_while_running_exit_zero", "relaunch_no_second_goose", "relaunch_goose_window_exists"): + for check in ("relaunch_while_running_exit_zero", "relaunch_no_second_goose", "relaunch_goose_window_exists", + "relaunch_goose_window_raised"): assert check in source, check From 0845f2064717aa5d01dcd029254608a5a429caba Mon Sep 17 00:00:00 2001 From: castrojo Date: Wed, 7 Oct 2026 22:09:16 -0400 Subject: [PATCH 19/19] fix(argo): ignore registry allowlist for golang build image --- argo/workflow-templates/run-chairlift-wayland-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml index 3064f4cbac..1c40c8e8f6 100644 --- a/argo/workflow-templates/run-chairlift-wayland-tests.yaml +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -98,7 +98,7 @@ spec: # glib-compile-schemas is the one the application will read them with; # `make schemas` skips cleanly here because the builder lacks it. - name: build-chairlift - image: golang:1.26 + image: golang:1.26 # registry-lint-ignore: Go toolchain for in-pod build, pulled from docker.io through local Zot cache resources: requests: cpu: "2"