diff --git a/argo/workflow-templates/run-chairlift-wayland-tests.yaml b/argo/workflow-templates/run-chairlift-wayland-tests.yaml new file mode 100644 index 0000000000..1c40c8e8f6 --- /dev/null +++ b/argo/workflow-templates/run-chairlift-wayland-tests.yaml @@ -0,0 +1,1429 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: run-chairlift-wayland-tests + namespace: argo + annotations: + description: | + Builds ChairLift from a branch and runs its behave + dogtail AT-SPI suite + (test/e2e/features) inside a live GNOME Wayland session: the nested, + systemd-booted Dakota target run-container-tests boots (headless GNOME + Shell with a virtual monitor, GDM autologin, test-user linger), driven by + qecore-headless --session-type wayland. The application (and Goose's + Electron window) run with GDK_BACKEND=wayland and DISPLAY unset. On x86_64 it + then installs Enhanced Troubleshooting's packages through ChairLift's own + troubleshoot.Setup, with no Homebrew on PATH, and checks that Goose + desktop launched in ChairLift's profile writes nothing outside it. + labels: + app.kubernetes.io/component: test-runner + app.kubernetes.io/part-of: bluefin-test-suite +spec: + serviceAccountName: argo + entrypoint: run-chairlift-wayland-tests + podGC: + strategy: OnWorkflowSuccess + ttlStrategy: + secondsAfterSuccess: 604800 + secondsAfterFailure: 2592000 + arguments: + parameters: + - name: chairlift-repo + value: "https://github.com/projectbluefin/chairlift" + - name: chairlift-branch + value: "main" + - name: image + value: "ghcr.io/projectbluefin/dakota" + - name: image-tag + value: "testing" + - name: behave-tags + value: "" + - name: goose-isolation-check + value: "true" + # false: only the smallest model's tool-call check, no larger models. + - name: answer-quality + value: "true" + templates: + - name: run-chairlift-wayland-tests + # Same placement as run-container-tests: ghost is the only node with a + # graphical seat and the canonical local-path result store. + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + kubernetes.io/hostname: ghost + # `make ci` and the build (~15m) + pull (≤34m worst case, normally + # minutes) + boot and pip (~5m) + one application launch per scenario + # (~12m) + the engine check: installs, Agent Mode, two launches, and up + # to three model pulls and CLI answers (≤15m each), and the screenshots. + activeDeadlineSeconds: 7200 + # Leaf template holding the shared ghost QA capacity key, like + # run-container-tests: each run boots one nested GNOME target. + synchronization: + semaphores: + - configMapKeyRef: + name: workflow-semaphores + key: ghost-container-qa + inputs: + parameters: + - name: chairlift-repo + value: "{{workflow.parameters.chairlift-repo}}" + - name: chairlift-branch + value: "{{workflow.parameters.chairlift-branch}}" + - name: image + value: "{{workflow.parameters.image}}" + - name: image-tag + value: "{{workflow.parameters.image-tag}}" + - name: behave-tags + value: "{{workflow.parameters.behave-tags}}" + - name: goose-isolation-check + value: "{{workflow.parameters.goose-isolation-check}}" + - name: answer-quality + value: "{{workflow.parameters.answer-quality}}" + outputs: + parameters: + - name: result + valueFrom: + path: /tmp/results/result-summary.txt + default: "No results generated" + - name: failed-scenarios + valueFrom: + path: /tmp/results/failed-scenarios.json + default: "[]" + - name: goose-isolation + valueFrom: + path: /tmp/results/goose/verdict.json + default: "{}" + initContainers: + # The Go build happens in the pod, before the target exists, so the + # suite runs the branch's own chairlift_e2e binary and nothing else. + # Schemas are compiled later inside the Dakota target, whose + # glib-compile-schemas is the one the application will read them with; + # `make schemas` skips cleanly here because the builder lacks it. + - name: build-chairlift + image: golang:1.26 # registry-lint-ignore: Go toolchain for in-pod build, pulled from docker.io through local Zot cache + resources: + requests: + cpu: "2" + memory: 4Gi + limits: + cpu: "4" + memory: 8Gi + volumeMounts: + - mountPath: /workspace + name: workspace + env: + - name: CHAIRLIFT_REPO + value: "{{inputs.parameters.chairlift-repo}}" + - name: CHAIRLIFT_BRANCH + value: "{{inputs.parameters.chairlift-branch}}" + command: [bash, -c] + args: + - | + set -euo pipefail + # A missing branch must fail: never report results from other code. + git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}" /workspace/chairlift + cd /workspace/chairlift + mkdir -p /workspace/lane/bin /workspace/lane/inventory + git rev-parse HEAD | tee /workspace/lane/chairlift-commit + + # `make ci`, ChairLift's whole host-independent CI gate, on the + # untouched checkout (before any lane tool is written into it). Its + # result is recorded and reported; the lane goes on either way, so + # one run answers both questions. golangci-lint is the version CI + # pins (.github/workflows/test.yml), fetched by checksum. + LINT_VERSION=2.12.2 + LINT_SHA256=8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553 + LINT_TGZ="golangci-lint-${LINT_VERSION}-linux-amd64.tar.gz" + curl -sSfL -o "/tmp/${LINT_TGZ}" \ + "https://github.com/golangci/golangci-lint/releases/download/v${LINT_VERSION}/${LINT_TGZ}" + echo "${LINT_SHA256} /tmp/${LINT_TGZ}" | sha256sum -c - + tar -xzf "/tmp/${LINT_TGZ}" -C /tmp + install -m 0755 "/tmp/golangci-lint-${LINT_VERSION}-linux-amd64/golangci-lint" /usr/local/bin/golangci-lint + CI_RC=0 + make ci >/workspace/lane/make-ci.log 2>&1 || CI_RC=$? + echo "${CI_RC}" >/workspace/lane/make-ci-rc + echo "make ci exit ${CI_RC}"; tail -n 40 /workspace/lane/make-ci.log + git status --porcelain + + CGO_ENABLED=0 make build-e2e schemas + # What a Homebrew install puts on disk: the GUI and the wrapper the + # desktop entry and the distro's Ask Bluefin menu entry run. + mkdir -p /workspace/lane/install + install -m 0755 build/chairlift /workspace/lane/install/chairlift + install -m 0755 data/chairlift-wrapper.sh /workspace/lane/install/chairlift-wrapper + + # Lane tools live inside the module so they may import internal/. + # navjson: the page and shortcut inventories the Go gate + # (test/e2e/atspi_behave_test.go) hands the suite, from the one + # authority, internal/navigation. + mkdir -p build/lane/navjson build/lane/troubleshootcheck + cat >build/lane/navjson/main.go <<'GO' + package main + + import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/projectbluefin/chairlift/internal/branding" + "github.com/projectbluefin/chairlift/internal/navigation" + ) + + type item struct { + Name string `json:"name"` + Title string `json:"title"` + } + + func write(dir, name string, value any) { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0o644); err != nil { + panic(err) + } + } + + func main() { + dir := os.Args[1] + items := navigation.Items() + nav := make([]item, 0, len(items)) + for _, it := range items { + nav = append(nav, item{Name: it.Name, Title: it.Title}) + } + var shortcuts []string + for _, s := range navigation.Shortcuts(items) { + shortcuts = append(shortcuts, s.Title) + } + write(dir, "navigation.json", nav) + write(dir, "shortcuts.json", shortcuts) + if err := os.WriteFile(filepath.Join(dir, "app-name.txt"), []byte(branding.AppName), 0o644); err != nil { + panic(err) + } + } + GO + # troubleshootcheck: drives the branch's own internal/troubleshoot. + # setup troubleshoot.Setup(troubleshoot.Detect(), progress): + # the tap and the three installs, through + # internal/homebrew exactly as the app runs them. + # agent-mode aistack.Enable, then aistack.WaitHealthy. + # model resolves through aistack's live resolver, + # ranks it, then aistack.PullModel and + # aistack.ConfigureActiveModel (the alias path). + # command-path the PATH troubleshoot.Command hands the launch. + # write troubleshoot.ProfileAt(dir).Write(server). + # enabled the extensions a config.yaml leaves enabled. + cat >build/lane/troubleshootcheck/main.go <<'GO' + package main + + import ( + "context" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/projectbluefin/chairlift/internal/aistack" + "github.com/projectbluefin/chairlift/internal/troubleshoot" + "gopkg.in/yaml.v3" + ) + + func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + func printState(s troubleshoot.State) { + fmt.Printf("supported=%t\nserver=%s\ndesktop=%s\nllmman=%s\n", s.Supported, s.ServerPath, s.DesktopPath, s.LLMManPath) + } + + func main() { + switch { + case len(os.Args) == 2 && os.Args[1] == "setup": + fmt.Println("PATH=" + os.Getenv("PATH")) + state, err := troubleshoot.Setup(troubleshoot.Detect(), func(step string) { + fmt.Println("progress: " + step) + }) + printState(state) + if err != nil { + fail(err) + } + if !state.Installed() { + fail(fmt.Errorf("setup returned without every package installed")) + } + case len(os.Args) == 2 && os.Args[1] == "agent-mode": + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute) + defer cancel() + fmt.Println("PATH=" + os.Getenv("PATH")) + start := time.Now() + if err := aistack.Enable(ctx); err != nil { + fmt.Printf("enable_seconds=%.0f\n", time.Since(start).Seconds()) + fail(err) + } + fmt.Printf("enable_seconds=%.0f\nllmman=%s\n", time.Since(start).Seconds(), aistack.Executable()) + if !aistack.WaitHealthy(ctx, 3*time.Minute) { + fail(errors.New("llmman never answered /llmman/node")) + } + node, err := aistack.FetchNodeStatus(ctx) + if err != nil { + fail(err) + } + fmt.Printf("healthy=true\nnode_memory=%d\n", node.Memory) + case len(os.Args) == 3 && os.Args[1] == "model": + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + node, err := aistack.FetchNodeStatus(ctx) + if err != nil { + fail(err) + } + live, err := aistack.ResolveFamilyLive(ctx, aistack.FamilyQwen, os.Args[2], aistack.DefaultFetch) + if err != nil { + fail(err) + } + ranked := aistack.RankCandidateModels(aistack.FilterEligibleModels(live, node.Memory)) + if len(ranked) == 0 { + fail(fmt.Errorf("no eligible file in %s for %d bytes of memory", os.Args[2], node.Memory)) + } + choice := ranked[0] + ref := choice.ModelRef() + fmt.Printf("ref=%s\nfile=%s\nquant=%s\nsize_bytes=%d\n", ref, choice.File, choice.Quant, choice.SizeBytes) + start := time.Now() + if err := aistack.PullModel(ctx, ref); err != nil { + fail(err) + } + fmt.Printf("pull_seconds=%.0f\n", time.Since(start).Seconds()) + if err := aistack.ConfigureActiveModel(ctx, ref); err != nil { + fail(err) + } + active, err := aistack.ReadActiveModel(ctx) + if err != nil { + fail(err) + } + fmt.Println("active=" + active) + // llmman answers with the registry-qualified name it stored the + // model under (hf.co/), observed in chairlift-wayland-lane-r5vxf. + if active != ref && !strings.HasSuffix(active, "/"+ref) { + fail(fmt.Errorf("alias %s reads %q, want %q", aistack.ActiveModelAlias, active, ref)) + } + case len(os.Args) == 3 && os.Args[1] == "command-path": + cmd, err := troubleshoot.Command(troubleshoot.Detect(), troubleshoot.ProfileAt(os.Args[2]), aistack.ActiveModelAlias) + if err != nil { + fail(err) + } + fmt.Println("argv=" + strings.Join(cmd.Args, " ")) + for _, kv := range cmd.Env { + if strings.HasPrefix(kv, "PATH=") || strings.HasPrefix(kv, "GOOSE_PATH_ROOT=") || strings.HasPrefix(kv, "XDG_CONFIG_HOME=") { + fmt.Println(kv) + } + } + case len(os.Args) == 4 && os.Args[1] == "write": + profile := troubleshoot.ProfileAt(os.Args[2]) + if err := profile.Write(os.Args[3]); err != nil { + fail(err) + } + fmt.Println(profile.ConfigPath()) + case len(os.Args) == 3 && os.Args[1] == "enabled": + data, err := os.ReadFile(os.Args[2]) + if err != nil { + fail(err) + } + var cfg struct { + Extensions map[string]struct { + Enabled bool `yaml:"enabled"` + } `yaml:"extensions"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + fail(err) + } + var names []string + for name, ext := range cfg.Extensions { + if ext.Enabled { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + fmt.Println(name) + } + default: + fmt.Fprintln(os.Stderr, "usage: troubleshootcheck setup | agent-mode | model | command-path | write | enabled ") + os.Exit(2) + } + } + GO + CGO_ENABLED=0 go build -o /workspace/lane/bin/navjson ./build/lane/navjson + CGO_ENABLED=0 go build -o /workspace/lane/bin/troubleshootcheck ./build/lane/troubleshootcheck + /workspace/lane/bin/navjson /workspace/lane/inventory + cat /workspace/lane/inventory/navigation.json; echo + chmod -R a+rX /workspace + ls -l build/e2e/chairlift /workspace/lane/bin + script: + image: ghcr.io/projectbluefin/arc-runner:latest + imagePullPolicy: IfNotPresent + securityContext: + # Podman needs a writable cgroup hierarchy to boot the nested systemd host. + privileged: true + runAsUser: 0 + allowPrivilegeEscalation: true + resources: + requests: + cpu: "2" + memory: 4Gi + ephemeral-storage: 12Gi + limits: + cpu: "4" + memory: 8Gi + ephemeral-storage: 24Gi + env: + - name: IMAGE_REPO + value: "{{inputs.parameters.image}}" + - name: IMAGE_TAG + value: "{{inputs.parameters.image-tag}}" + - name: BEHAVE_TAGS + value: "{{inputs.parameters.behave-tags}}" + - name: GOOSE_CHECK + value: "{{inputs.parameters.goose-isolation-check}}" + - name: ANSWER_QUALITY + value: "{{inputs.parameters.answer-quality}}" + volumeMounts: + - mountPath: /workspace + name: workspace + - mountPath: /tmp/results + name: results + - mountPath: /var/lib/containers + name: containers-storage + - mountPath: /run/containers + name: containers-runroot + - mountPath: /run/udev + name: udev + - mountPath: /var/mnt/ghost-data/test-results + name: results-store + - mountPath: /var/cache/bluefin-qa-pip + name: pip-cache + command: [bash] + source: | + set -euo pipefail + TARGET_NAME="chairlift-wayland-target" + RESULT_DIR="/var/mnt/ghost-data/test-results/{{workflow.name}}/chairlift-wayland" + mkdir -p /tmp/results "${RESULT_DIR}" /opt/qa-wheels /var/cache/bluefin-qa-pip + + persist_results() { + mkdir -p "${RESULT_DIR}" + cp -a /tmp/results/. "${RESULT_DIR}/" || { + echo "ERROR: failed to persist results to ${RESULT_DIR}" >&2 + return 1 + } + touch "${RESULT_DIR}/.complete" + } + cleanup_target() { + podman rm --force "${TARGET_NAME}" >/dev/null 2>&1 || true + } + on_exit() { + local rc=$? + local persist_rc=0 + persist_results || persist_rc=$? + cleanup_target + if [[ "${persist_rc}" -ne 0 && "${rc}" -eq 0 ]]; then + rc="${persist_rc}" + fi + return "${rc}" + } + trap on_exit EXIT + # activeDeadlineSeconds and `argo stop` arrive as signals; an untrapped + # SIGTERM would skip the EXIT trap and leave the privileged target up. + trap 'exit 143' TERM + trap 'exit 130' INT + + command -v podman >/dev/null || { + echo "Podman is required for nested systemd QA" >&2 + exit 1 + } + echo "ChairLift commit: $(cat /workspace/lane/chairlift-commit)" | tee /tmp/results/chairlift-commit.txt >&2 + + # --- target image: through the local Zot pull-through cache -------- + zot_image() { + case "$1" in + ghcr.io/*) printf '192.168.1.102:30501/ghcr/%s' "${1#ghcr.io/}" ;; + quay.io/*) printf '192.168.1.102:30501/quay/%s' "${1#quay.io/}" ;; + docker.io/*) printf '192.168.1.102:30501/docker/%s' "${1#docker.io/}" ;; + *) printf '%s' "$1" ;; + esac + } + ZOT_IMAGE_REPO=$(zot_image "${IMAGE_REPO}") + PULL_TLS_ARGS=() + case "${ZOT_IMAGE_REPO}" in + 192.168.1.102:*|localhost:*|127.0.0.1:*) + mkdir -p /etc/containers/registries.conf.d + printf '%s\n' '[[registry]]' 'location = "192.168.1.102:30501"' 'insecure = true' \ + >/etc/containers/registries.conf.d/bluefin-local-zot.conf + PULL_TLS_ARGS=(--tls-verify=false) + ;; + esac + DIGEST=$(timeout 120 skopeo inspect "${PULL_TLS_ARGS[@]}" --no-tags \ + --format '{{.Digest}}' "docker://${ZOT_IMAGE_REPO}:${IMAGE_TAG}" 2>/dev/null || true) + if [[ -n "${DIGEST}" ]]; then + TARGET_IMAGE="${ZOT_IMAGE_REPO}@${DIGEST}" + else + echo "Warning: could not resolve a digest for ${IMAGE_REPO}:${IMAGE_TAG}; using the tag." >&2 + TARGET_IMAGE="${ZOT_IMAGE_REPO}:${IMAGE_TAG}" + fi + echo "${IMAGE_REPO}:${IMAGE_TAG} ${DIGEST:-unresolved}" | tee /tmp/results/target-image.txt >&2 + cat >/tmp/storage.conf <<'EOF' + [storage] + driver = "overlay" + graphroot = "/var/lib/containers/storage" + runroot = "/run/containers/storage" + EOF + export CONTAINERS_STORAGE_CONF=/tmp/storage.conf + # Bounded attempts: the Zot cache occasionally stalls mid-blob, and + # podman keeps finished blobs across attempts (see run-container-tests). + pull_ok=false + for attempt in 1 2 3 4; do + echo "Pulling ${TARGET_IMAGE} (attempt ${attempt}/4)..." >&2 + if timeout 480 podman pull "${PULL_TLS_ARGS[@]}" "${TARGET_IMAGE}" >/dev/null; then + pull_ok=true + break + fi + sleep 30 + done + [[ "${pull_ok}" == true ]] || { echo "Giving up on ${TARGET_IMAGE}" >&2; exit 1; } + + # --- boot the nested systemd/GDM target ---------------------------- + # /usr/share/chairlift and /etc/chairlift are masked: a shipped + # config.yml outranks every configuration fixture, and the suite's + # before_all refuses to run against it. /proc/cmdline is blanked so a + # composefs host's command line cannot send the real bootc reader to + # host state (the same isolation test/e2e/dakota_atspi.sh applies). + podman run --detach --systemd=always --network host --privileged \ + --cgroupns=host \ + --security-opt seccomp=unconfined \ + --tmpfs /run:rw,nosuid,nodev,mode=755,size=256m \ + --tmpfs /usr/share/chairlift:ro,notmpcopyup \ + --tmpfs /etc/chairlift:ro,notmpcopyup \ + --mount type=bind,source=/dev/null,destination=/proc/cmdline,ro \ + --name "${TARGET_NAME}" \ + --pull=never \ + --mount type=bind,source=/sys,destination=/sys,ro,bind-propagation=rprivate \ + --mount type=bind,source=/run/udev,destination=/run/udev,ro,bind-propagation=rprivate \ + --volume /workspace:/workspace:ro \ + --volume /tmp/results:/tmp/results \ + --volume /opt/qa-wheels:/var/opt/qa-wheels:ro \ + --volume /var/cache/bluefin-qa-pip:/var/cache/bluefin-qa-pip:rw \ + --volume /etc/resolv.conf:/etc/resolv.conf:ro \ + "${TARGET_IMAGE}" /sbin/init >/dev/null + + ready=false + for i in $(seq 1 60); do + state=$(podman exec "${TARGET_NAME}" systemctl is-system-running 2>/dev/null || true) + dbus_active=$(podman exec "${TARGET_NAME}" systemctl is-active dbus 2>/dev/null || true) + logind_active=$(podman exec "${TARGET_NAME}" systemctl is-active systemd-logind 2>/dev/null || true) + can_graphical=$(podman exec "${TARGET_NAME}" loginctl show-seat seat0 --property=CanGraphical 2>/dev/null || true) + echo "systemd readiness probe ${i}: state=${state:-unknown} dbus=${dbus_active:-unknown} logind=${logind_active:-unknown} seat0=${can_graphical:-unknown}" + if [[ "${state}" == "running" || "${state}" == "degraded" ]] \ + && [[ "${dbus_active}" == "active" ]] \ + && [[ "${logind_active}" == "active" ]] \ + && [[ "${can_graphical}" == "CanGraphical=yes" ]]; then + ready=true + break + fi + sleep 2 + done + if [[ "${ready}" != true ]]; then + echo "Nested systemd host did not reach a usable session state" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -n 300 || true + exit 1 + fi + + echo "Provisioning nested target..." >&2 + # Fed over stdin through a quoted heredoc, never `bash -c '...'` + # (see run-container-tests: an inner apostrophe truncates the latter). + setup_rc=0 + podman exec -i --env GOOSE_CHECK="${GOOSE_CHECK}" "${TARGET_NAME}" bash -s <<'NESTED_SETUP' || setup_rc=$? + set -euo pipefail + mkdir -p /run/dbus /run/systemd/seats /run/systemd/users + systemctl stop gdm || true + rm -rf /run/gdm /var/log/gdm + mkdir -p /run/gdm /var/log/gdm + getent group gdm >/dev/null 2>&1 || groupadd -r gdm + id -u gdm >/dev/null 2>&1 || useradd -r -g gdm -d /var/lib/gdm -s /usr/sbin/nologin gdm + chown -R gdm:gdm /run/gdm /var/log/gdm + chmod 755 /run/gdm + chmod 750 /var/log/gdm + + # Local group entries for the device groups; NSS-only groups are + # materialised because useradd accepts only /etc/group entries. + append_group_line() { + if [ -s /etc/group ] && [ "$(tail -c 1 /etc/group | wc -l)" -eq 0 ]; then + printf '\n' >>/etc/group + fi + printf '%s\n' "$1" >>/etc/group + } + # wheel: Bluefin's first user is an administrator. Dakota's + # org.freedesktop.Flatpak.rules lets an active, local wheel member + # install system Flatpaks without a prompt, and Agent Mode's + # Brewfile installs Jan that way ("Flatpak system operation Deploy + # not allowed for user" without it, chairlift-wayland-lane-q5xpf). + for group in video render input wheel; do + if ! grep -q "^${group}:" /etc/group; then + if ! groupadd -r "${group}" 2>/dev/null; then + nss_entry=$(getent group "${group}" || true) + if [ -n "${nss_entry}" ]; then + append_group_line "${nss_entry}" + else + case "${group}" in video) gid=39 ;; render) gid=989 ;; wheel) gid=10 ;; *) gid=999 ;; esac + append_group_line "${group}:x:${gid}:" + fi + fi + fi + grep -q "^${group}:" /etc/group || { echo "Failed to create local group ${group}" >&2; exit 1; } + done + if ! id bluefin-test >/dev/null 2>&1; then + useradd -m -u 1000 -G video,render,input,wheel bluefin-test + else + usermod -aG video,render,input,wheel bluefin-test + fi + # Synthetic input: the container's own /dev/uinput, given to the + # lane-local input group (see run-container-tests). + [ -e /dev/uinput ] || modprobe uinput 2>/dev/null || true + if [ -e /dev/uinput ]; then + chgrp input /dev/uinput && chmod 0660 /dev/uinput + else + echo "WARNING: /dev/uinput is absent" >&2 + fi + # GDM PAM autologin needs a non-expired shadow lastchg. + if command -v chage >/dev/null 2>&1; then + chage -d "$(date +%Y-%m-%d)" bluefin-test + else + today=$(( $(date +%s) / 86400 )) + sed -i "s/^bluefin-test:!*/bluefin-test:!:${today}:0:99999:7:::/" /etc/shadow + fi + mkdir -p /etc/sudoers.d + echo "bluefin-test ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/bluefin-test + chmod 0440 /etc/sudoers.d/bluefin-test + mkdir -p /etc/gdm + printf "%s\n" "[daemon]" "AutomaticLoginEnable=True" "AutomaticLogin=bluefin-test" \ + "InitialSetupEnable=False" >/etc/gdm/custom.conf + mkdir -p /tmp/results/atspi /tmp/results/goose + chown -R 1000:1000 /home/bluefin-test /tmp/results + + # Headless shell, never DRM master; --unsafe-mode must be repeated + # because this ExecStart= reset overrides qecore's own edit. + mkdir -p /etc/systemd/user/org.gnome.Shell@.service.d + printf "%s\n" "[Service]" "ExecStart=" \ + "ExecStart=/usr/bin/gnome-shell --mode=%i --unsafe-mode --headless --virtual-monitor 1920x1080" \ + >/etc/systemd/user/org.gnome.Shell@.service.d/10-headless.conf + + # Dakota's session pulls in brew-preinstall.service (a user unit + # that installs the preinstall.d Brewfiles over the network), and + # its brew-update/upgrade timers run against the same prefix. None + # is under test, and each would contend with the Goose check's + # installs for Homebrew's locks. Masked before any user manager + # starts, since a manager reads unit files only at start. + systemctl --global mask brew-preinstall.service >/dev/null 2>&1 || true + systemctl mask --runtime brew-update.timer brew-upgrade.timer >/dev/null 2>&1 || true + + loginctl enable-linger bluefin-test + # logind sizes the user slice's TasksMax from the container's pids + # budget; a desktop has orders of magnitude more headroom. + systemctl set-property --runtime user-1000.slice TasksMax=infinity || true + for _ in $(seq 1 30); do [ -S /run/user/1000/bus ] && break; sleep 1; done + if [ ! -S /run/user/1000/bus ]; then + echo "user@1000.service did not create /run/user/1000/bus after enable-linger" >&2 + loginctl show-user bluefin-test >&2 || true + systemctl status user@1000.service --no-pager --full >&2 || true + exit 1 + fi + # GNOME Shell's first-login "Welcome to Bluefin / Take Tour" dialog + # is a Shell modal: while it is up it takes every key, so the + # suite's input reaches no window even though the window holds + # focus (observed: a whole run of keyboard steps failed behind + # it). Mark it shown in the test user's own dconf before any + # session starts. + sudo -u bluefin-test env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + gsettings set org.gnome.shell welcome-dialog-last-shown-version "'999'" + sudo -u bluefin-test env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + gsettings get org.gnome.shell welcome-dialog-last-shown-version >&2 + + # qecore-headless (system interpreter; the target is disposable). + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --default-pip >&2 + export PIP_CACHE_DIR=/var/cache/bluefin-qa-pip + pip_args=(--disable-pip-version-check --no-input --cache-dir "${PIP_CACHE_DIR}") + if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then + pip_args+=(--break-system-packages) + fi + # setuptools<81: qecore imports pkg_resources (see run-container-tests). + qa_dependencies=("setuptools<81" qecore) + pip_ok=false + for attempt in 1 2 3; do + if python3 -m pip install "${pip_args[@]}" --find-links /var/opt/qa-wheels \ + "${qa_dependencies[@]}" >/tmp/pip-install.log 2>&1; then + pip_ok=true + break + fi + echo "pip install attempt ${attempt}/3 failed:" >&2 + tail -n 40 /tmp/pip-install.log >&2 + sleep 10 + done + [ "${pip_ok}" = true ] || { cat /tmp/pip-install.log >&2; exit 1; } + # python-uinput is qecore's uinput probe; Wayland input goes through + # gnome-ponytail-daemon, so a failed build here is a warning only. + python3 -m pip install "${pip_args[@]}" --find-links /var/opt/qa-wheels python-uinput \ + >/tmp/pip-uinput.log 2>&1 || { echo "WARNING: python-uinput did not install" >&2; tail -n 20 /tmp/pip-uinput.log >&2; } + # qecore-headless dconf ownership TOCTOU (see run-container-tests). + headless_script=$(command -v qecore-headless || true) + [ -n "${headless_script}" ] || { echo "qecore-headless not found" >&2; exit 1; } + if ! grep -q 'dconf/user || true"' "${headless_script}"; then + matches=$(grep -c '/dconf/user"$' "${headless_script}" || true) + [ "${matches}" = "1" ] || { echo "Unexpected qecore-headless layout (${matches})" >&2; exit 1; } + sed -i 's#/dconf/user"$#/dconf/user || true"#' "${headless_script}" + fi + python3 -c "import ast,sys; ast.parse(open(sys.argv[1]).read())" "${headless_script}" + + # The suite's own interpreter: a venv over the system site-packages + # (PyGObject, pyatspi, ponytail come from the image) with ChairLift's + # hash-pinned behave and dogtail, exactly as dakota_atspi.sh builds it. + python3 -m venv --system-site-packages /home/bluefin-test/atspi-venv + /home/bluefin-test/atspi-venv/bin/python -m pip install -q --disable-pip-version-check \ + --require-hashes -r /workspace/chairlift/test/e2e/requirements-atspi.txt + # Compile the branch's GSettings schemas with the image's own tool. + mkdir -p /home/bluefin-test/chairlift-schemas + cp /workspace/chairlift/data/*.gschema.xml /home/bluefin-test/chairlift-schemas/ + glib-compile-schemas /home/bluefin-test/chairlift-schemas + chown -R 1000:1000 /home/bluefin-test + + # The Goose check installs through the image's Homebrew, which + # brew-setup.service extracts on first boot. Assert the binary, not + # the unit's exit code (a satisfied Condition* also exits 0). + if [ "${GOOSE_CHECK}" = "true" ] && [ "$(uname -m)" = "x86_64" ]; then + brew_bin="" + for candidate in /home/linuxbrew/.linuxbrew/bin/brew /var/home/linuxbrew/.linuxbrew/bin/brew; do + [ -x "${candidate}" ] && brew_bin="${candidate}" && break + done + if [ -z "${brew_bin}" ]; then + systemctl unmask --runtime brew-setup.service || true + brew_setup_rc=0 + systemctl start brew-setup.service || brew_setup_rc=$? + for candidate in /home/linuxbrew/.linuxbrew/bin/brew /var/home/linuxbrew/.linuxbrew/bin/brew; do + [ -x "${candidate}" ] && brew_bin="${candidate}" && break + done + if [ -z "${brew_bin}" ]; then + echo "No Homebrew after brew-setup.service (start exit ${brew_setup_rc})" >&2 + systemctl status brew-setup.service --no-pager --full >&2 || true + exit 1 + fi + fi + echo "${brew_bin}" >/tmp/results/goose/brew-path.txt + ls -ld "$(dirname "$(dirname "$(dirname "${brew_bin}")")")" "$(dirname "$(dirname "${brew_bin}")")" >&2 + fi + + systemctl start gdm || { systemctl status gdm --no-pager --full >&2 || true; exit 1; } + echo "Nested target provisioning complete." >&2 + NESTED_SETUP + if [ "${setup_rc}" -ne 0 ]; then + echo "Nested target provisioning failed (rc=${setup_rc})" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -n 200 >&2 || true + exit 1 + fi + for _ in $(seq 1 60); do + podman exec "${TARGET_NAME}" pgrep -u 1000 -f gnome-session >/dev/null && break + sleep 2 + done + podman exec "${TARGET_NAME}" pgrep -u 1000 -f gnome-session >/dev/null || { + echo "GDM did not start a GNOME session for bluefin-test" >&2 + podman exec "${TARGET_NAME}" journalctl --no-pager -u gdm >&2 || true + podman exec "${TARGET_NAME}" journalctl --no-pager _COMM=gnome-shell >&2 || true + exit 1 + } + + echo "=== nested target environment snapshot ===" >&2 + podman exec "${TARGET_NAME}" bash -c 'tr "\0" " " &2 || true + podman exec "${TARGET_NAME}" bash -c 'ls /run/user/1000; ls -l /dev/uinput; ls /usr/libexec/gnome-ponytail-daemon; python3 -c "import ponytail.ponytail; print(\"ponytail module ok\")"; cat /usr/lib/os-release | head -3' >&2 || true + echo "=== end environment snapshot ===" >&2 + + # behave tags reach the in-session script through a file: + # qecore-headless replaces its environment with the session's. + printf '%s' "${BEHAVE_TAGS}" >/workspace/lane/behave-tags + printf '%s' "${GOOSE_CHECK}" >/workspace/lane/goose-check + printf '%s' "${ANSWER_QUALITY}" >/workspace/lane/answer-quality + cat >/workspace/lane/run-chairlift.sh <<'RUNNER' + #!/bin/bash + # Runs inside the bluefin-test GNOME Wayland session (qecore-headless). + set -uo pipefail + OUT=/tmp/results/atspi + LANE=/workspace/lane + echo "session: XDG_SESSION_TYPE=${XDG_SESSION_TYPE:-} WAYLAND_DISPLAY=${WAYLAND_DISPLAY:-} XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-} DISPLAY=${DISPLAY:-}" + + # Shell ready = Shell.Eval answers (needs --unsafe-mode) with a panel. + shell_ready=false + for attempt in $(seq 1 60); do + if gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.panel ? true : false' 2>/dev/null | grep -q "(true, 'true')"; then + shell_ready=true + break + fi + sleep 2 + done + if [ "${shell_ready}" != true ]; then + echo "GNOME Shell never answered Shell.Eval" >&2 + echo 3 >"${OUT}/behave-rc.txt" + exit 3 + fi + # No Shell modal may hold the keyboard when the suite starts. The + # session opens in the overview, which is itself a modal, so leave + # it first; what remains (the welcome dialog, a prompt) is a fault. + for attempt in $(seq 1 30); do + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.overview.hide(); Main.modalCount' 2>/dev/null | grep -q "(true, '0')" && break + sleep 1 + done + if ! gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell \ + --method org.gnome.Shell.Eval 'Main.modalCount' 2>/dev/null | grep -q "(true, '0')"; then + echo "a GNOME Shell modal is still holding the keyboard" >&2 + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${OUT}/shell-modal.png" >&2 || true + echo 6 >"${OUT}/behave-rc.txt" + exit 6 + fi + + + # Wayland only: the window must reach the session's compositor + # natively, never through the session's on-demand Xwayland. + # qecore-headless hands the script gnome-session's environment, which + # predates the compositor; gnome-shell exports WAYLAND_DISPLAY to the + # user manager once its socket is up, so read it from there. + if [ -z "${WAYLAND_DISPLAY:-}" ]; then + WAYLAND_DISPLAY=$(systemctl --user show-environment 2>/dev/null | sed -n 's/^WAYLAND_DISPLAY=//p') + fi + case "${WAYLAND_DISPLAY:-}" in + /*) WAYLAND_SOCKET="${WAYLAND_DISPLAY}" ;; + *) WAYLAND_SOCKET="${XDG_RUNTIME_DIR}/${WAYLAND_DISPLAY:-}" ;; + esac + if [ -z "${WAYLAND_DISPLAY:-}" ] || [ ! -S "${WAYLAND_SOCKET}" ]; then + echo "the session exposes no Wayland socket (WAYLAND_DISPLAY='${WAYLAND_DISPLAY:-}')" >&2 + echo 4 >"${OUT}/behave-rc.txt" + exit 4 + fi + # Absolute, as run_atspi.sh's wayland_session.sh exports it: each + # scenario gets its own XDG_RUNTIME_DIR, against which a bare + # "wayland-0" would resolve to nothing. + WAYLAND_DISPLAY="${WAYLAND_SOCKET}" + export WAYLAND_DISPLAY + echo "Wayland socket: ${WAYLAND_SOCKET}" + export XDG_SESSION_TYPE=wayland GDK_BACKEND=wayland + unset DISPLAY XAUTHORITY + # The same isolation run_atspi.sh applies, minus the display: memory + # GSettings, no portals, the accessibility bridge on. + export LANG=C LC_ALL=C GSETTINGS_BACKEND=memory GDK_DEBUG=no-portals GTK_A11Y=atspi + unset NO_AT_BRIDGE + export GSETTINGS_SCHEMA_DIR=/home/bluefin-test/chairlift-schemas + export CHAIRLIFT_REQUIRE_ATSPI=1 + export CHAIRLIFT_ATSPI_APP=/workspace/chairlift/build/e2e/chairlift + export CHAIRLIFT_ATSPI_OUT="${OUT}" + CHAIRLIFT_NAVIGATION="$(cat "${LANE}/inventory/navigation.json")" + CHAIRLIFT_SHORTCUTS="$(cat "${LANE}/inventory/shortcuts.json")" + CHAIRLIFT_APP_NAME="$(cat "${LANE}/inventory/app-name.txt")" + export CHAIRLIFT_NAVIGATION CHAIRLIFT_SHORTCUTS CHAIRLIFT_APP_NAME + PY=/home/bluefin-test/atspi-venv/bin/python + + # CHAIRLIFT_REQUIRE_ATSPI: a missing stack fails, it never skips. + if ! "${PY}" -c 'from dogtail.config import config; import behave, pyatspi, gi; gi.require_version("Gtk", "3.0"); gi.require_version("Gdk", "3.0"); import dogtail.hermetic.mutter'; then + echo "accessibility stack is unavailable in the Wayland session" >&2 + echo 5 >"${OUT}/behave-rc.txt" + exit 5 + fi + + TAG_ARGS=() + TAGS="$(cat "${LANE}/behave-tags")" + [ -n "${TAGS}" ] && TAG_ARGS=(--tags "${TAGS}") + cd /tmp/results + BEHAVE_RC=0 + timeout 2700 "${PY}" -m behave /workspace/chairlift/test/e2e/features \ + --no-capture --no-capture-stderr \ + --format pretty --outfile "${OUT}/behave.log" \ + --format json.pretty --outfile "${OUT}/results.json" \ + --format progress \ + --junit --junit-directory "${OUT}/junit" \ + "${TAG_ARGS[@]}" || BEHAVE_RC=$? + echo "${BEHAVE_RC}" >"${OUT}/behave-rc.txt" + + # ---- Goose desktop profile isolation (x86_64 only) ---------------- + G=/tmp/results/goose + if [ "$(cat "${LANE}/goose-check")" != "true" ] || [ "$(uname -m)" != "x86_64" ]; then + echo skipped >"${G}/rc.txt" + exit 0 + fi + BREW="$(cat "${G}/brew-path.txt")" + # ChairLift's own profile location: $XDG_DATA_HOME, else ~/.local/share. + DATA="${HOME}/.local/share" + PROFILE="${DATA}/chairlift/troubleshooting" + # Every ChairLift step runs the way GNOME runs an app it launched: a + # transient unit in the user manager's app.slice, inside the user's + # logind context. (This script itself sits in podman's exec cgroup, + # outside any session, so polkit would treat it as inactive.) + # The install goes through ChairLift itself — troubleshoot.Setup over + # internal/homebrew — with a PATH that deliberately lacks Homebrew's + # bin and no `brew shellenv`, which is how a direct `chairlift` launch + # runs it. ChairLift has to find brew through its fallback and hand + # brew's children Homebrew's bin itself: the goose-linux cask's + # preflight runs a bare `rpm2cpio | cpio` (exit 127 without it). + # Auto-update is off only to bound the lane's time. + # TasksMax: the user manager's default is a share of the container's + # pids budget, far below a desktop's; brew's download threads hit it + # ("can't create Thread", chairlift-wayland-lane-dsjsw). + BARE_ENV=(systemd-run --user --wait --pipe --quiet --collect --slice=app.slice -p TasksMax=infinity + -p "UnsetEnvironment=HOMEBREW_PREFIX HOMEBREW_CELLAR HOMEBREW_REPOSITORY" + --setenv=PATH=/usr/bin:/bin --setenv=HOMEBREW_NO_AUTO_UPDATE=1) + systemctl --user set-property --runtime app.slice TasksMax=infinity 2>/dev/null || true + for unit in user-1000.slice; do systemctl show "${unit}" -p TasksMax -p TasksCurrent; done >"${G}/tasks-limits.txt" 2>&1 + systemctl --user show app.slice -p TasksMax -p TasksCurrent >>"${G}/tasks-limits.txt" 2>&1 + SRC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" setup >"${G}/setup.log" 2>&1 "${G}/setup-rc.txt" + # (`command-path` runs after Agent Mode, once llmman really resolves.) + { + "${BREW}" list --versions linux-mcp-server cpio rpm2cpio + "${BREW}" list --cask --versions goose-linux + } >"${G}/brew-installed.txt" 2>&1 || true + if [ "${SRC}" -ne 0 ]; then + echo "setup-failed" >"${G}/rc.txt" + exit 0 + fi + SERVER="$(sed -n 's/^server=//p' "${G}/setup.log")" + ls -la "${HOME}/.config" >"${G}/home-config-before.txt" 2>&1 + + # Agent Mode for real, through ChairLift's aistack on the stock host: + # tap llmmanorg/tap, trust only the llmman formula, brew bundle, + # `llmman serve --pull-only`, the user unit, readiness on + # /llmman/node. Nothing here trusts anything on ChairLift's behalf. + # Then the smallest Qwen3 through the alias path. + # Where this process stands with logind and polkit: Agent Mode's Jan + # Flatpak is a system install, which Dakota's flatpak rule allows + # only for an active, local wheel member. + { + id + echo "cgroup: $(cat /proc/self/cgroup)" + loginctl list-sessions --no-legend + loginctl show-user "$(id -u)" -p Display -p Sessions -p State + for s in $(loginctl list-sessions --no-legend | awk '{print $1}'); do + loginctl show-session "${s}" -p Id -p Name -p Seat -p Type -p Class -p Active -p Remote -p State + done + echo "pkcheck (this script, podman exec):" + pkcheck --action-id org.freedesktop.Flatpak.app-install --process "${BASHPID}" 2>&1; echo "rc=$?" + echo "pkcheck (app unit, as ChairLift's steps run):" + "${BARE_ENV[@]}" bash -c 'cat /proc/self/cgroup; pkcheck --action-id org.freedesktop.Flatpak.app-install --process "$(cut -d" " -f4 /proc/self/stat)" 2>&1; echo rc=$?' &1 + } >"${G}/polkit-session.txt" 2>&1 + ARC=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" agent-mode >"${G}/agent-mode.log" 2>&1 "${G}/agent-mode-rc.txt" + # What Homebrew now trusts: the formula, not the whole tap. + "${BREW}" tap-info --json llmmanorg/tap >"${G}/tap-info.json" 2>"${G}/tap-info.err" || true + find "${HOME}" -maxdepth 3 -name trust.json -path '*homebrew*' 2>/dev/null | while read -r f; do + echo "== ${f}"; cat "${f}"; echo + done >"${G}/trust-state.txt" + systemctl --user status chairlift-llmman.service --no-pager >"${G}/llmman-unit.txt" 2>&1 || true + if [ "${ARC}" -ne 0 ]; then + echo "agent-mode-failed" >"${G}/rc.txt" + exit 0 + fi + LLMMAN="$(sed -n 's/^llmman=//p' "${G}/agent-mode.log")" + # Models through ChairLift's alias path: aistack.PullModel must + # succeed on its own (no lane fallback), then ConfigureActiveModel. + MODEL_REPOS=(unsloth/Qwen3-0.6B-GGUF unsloth/Qwen3-1.7B-GGUF unsloth/Qwen3-4B-GGUF) + model_step() { + local n="$1" rc=0 + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" model "${MODEL_REPOS[$((n - 1))]}" >"${G}/model-${n}.log" 2>&1 "${G}/model-${n}-rc.txt" + curl -s --max-time 5 http://127.0.0.1:17434/llmman/node >"${G}/node-${n}.json" 2>&1 || true + "${LLMMAN}" ls >"${G}/llmman-ls-${n}.txt" 2>&1 || true + return "${rc}" + } + if ! model_step 1; then + echo "model-failed" >"${G}/rc.txt" + exit 0 + fi + flatpak list --app --columns=application,installation >"${G}/flatpaks.txt" 2>&1 || true + "${BARE_ENV[@]}" "${LANE}/bin/troubleshootcheck" command-path "${DATA}" >"${G}/command-env.txt" 2>&1 /dev/null || sudo -n cat "/proc/${SHELL_PID}/environ") + MENU_PATH="$(printf '%s' "${SHELL_PATH:-/usr/bin:/bin}" | tr ':' '\n' | grep -v linuxbrew | paste -sd: -)" + { + echo "gnome_shell_pid=${SHELL_PID}" + echo "shell_path=${SHELL_PATH:-}" + echo "menu_path=${MENU_PATH}" + printf '%s\n' "${SHELL_ENV[@]}" | grep -E '^(WAYLAND_DISPLAY|DISPLAY|XDG_RUNTIME_DIR|XDG_SESSION_TYPE|DBUS_SESSION_BUS_ADDRESS|HOME|XDG_DATA_HOME|XDG_CONFIG_HOME)=' + } >"${G}/shell-env.txt" + + # One launch, observed while Goose runs: its process tree, the + # environment llmman handed it (read from Goose's own `goose serve` + # backend: the Electron browser process's environ does not carry it), + # where its single-instance lock is, and a screenshot. + observe_launch() { + local tag="$1"; shift + systemd-run --user --scope --quiet --collect --slice=app.slice -p TasksMax=infinity \ + env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}" "$@" >"${G}/${tag}.log" 2>&1 /dev/null | grep -q -- '--type=' || { main="${pid}"; break; } + done + goosed="$(pgrep -u "$(id -u)" -f '/usr/lib/Goose/resources/bin/goose serve' | head -1)" + if kill -0 "${launcher}" 2>/dev/null; then rc=running; else wait "${launcher}"; rc=$?; fi + { + echo "launcher_exit=${rc}" + echo "alive=$( [ -n "${main}" ] && kill -0 "${main}" 2>/dev/null && echo yes || echo no)" + echo "goose_main_pid=${main}" + echo "goosed_pid=${goosed}" + echo "goose_parent=$( [ -n "${main}" ] && ps -o ppid= -p "${main}" | tr -d ' ')" + echo "profile_lock=$(readlink "${PROFILE}/desktop/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + echo "home_lock=$(readlink "${HOME}/.config/Goose/SingletonLock" 2>/dev/null || echo ABSENT)" + } >"${G}/${tag}-lock.txt" + [ -n "${goosed}" ] && tr '\0' '\n' <"/proc/${goosed}/environ" \ + | grep -E '^(GOOSE_[A-Z_]*|OPENAI_HOST|OPENAI_BASE_PATH|XDG_CONFIG_HOME|WAYLAND_DISPLAY|DISPLAY)=' \ + | grep -v -E '^GOOSE_SERVER__SECRET_KEY=' >"${G}/${tag}-environ.txt" + [ -n "${main}" ] && tr '\0' ' ' <"/proc/${main}/cmdline" >"${G}/${tag}-cmdline.txt" + ps -eo pid,ppid,args | grep -i -E 'goose|electron|llmman|chairlift' | grep -v grep | cut -c1-400 >"${G}/${tag}-processes.txt" || true + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/${tag}.png" >"${G}/${tag}-screenshot.txt" 2>&1 || true + [ "${KEEP_GOOSE:-0}" = 1 ] || stop_goose + if [ "${rc}" = running ]; then + pkill -TERM -u "$(id -u)" -x chairlift 2>/dev/null + wait "${launcher}" 2>/dev/null + fi + } + stop_goose() { + pkill -TERM -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + sleep 5 + pkill -KILL -u "$(id -u)" -f '/usr/lib/Goose/Goose' 2>/dev/null + for _ in $(seq 1 20); do [ -L "${PROFILE}/desktop/Goose/SingletonLock" ] || break; sleep 0.5; done + } + goose_mains() { + local pid + for pid in $(pgrep -u "$(id -u)" -f '/usr/lib/Goose/Goose'); do + tr '\0' ' ' <"/proc/${pid}/cmdline" 2>/dev/null | grep -q -- '--type=' || echo "${pid}" + done + } + goose_windows() { + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell --method org.gnome.Shell.Eval \ + "let ws = global.get_window_actors().map(a => a.meta_window).filter(w => /goose/i.test(w.get_wm_class() || '')); let f = global.display.focus_window; JSON.stringify({windows: ws.length, minimized: ws.filter(w => w.minimized).length, focused: !!f && /goose/i.test(f.get_wm_class() || '')})" 2>&1 + } + # (3) A cold `chairlift --ask-bluefin`; Goose is left running. + KEEP_GOOSE=1 observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin + # (6) The menu again while that session runs: Goose's single-instance + # lock must bring the running window forward rather than start a + # second Goose. The window is minimized first, so "brought forward" + # is visible as unminimized and focused. + { + echo "mains_before=$(goose_mains | paste -sd, -)" + gdbus call --session --dest org.gnome.Shell --object-path /org/gnome/Shell --method org.gnome.Shell.Eval \ + "global.get_window_actors().map(a => a.meta_window).filter(w => /goose/i.test(w.get_wm_class() || '')).forEach(w => w.minimize()); true" >/dev/null 2>&1 + sleep 1 + echo "windows_before=$(goose_windows)" + RRC=0 + timeout 60 systemd-run --user --scope --quiet --collect --slice=app.slice -p TasksMax=infinity \ + env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}" bash -c "${BIN}/chairlift-wrapper --ask-bluefin" \ + >"${G}/relaunch.log" 2>&1 "${G}/relaunch-state.txt" 2>&1 + gdbus call --session --dest org.gnome.Shell.Screenshot --object-path /org/gnome/Shell/Screenshot \ + --method org.gnome.Shell.Screenshot.Screenshot false false "${G}/relaunch.png" >/dev/null 2>&1 || true + stop_goose + # (5) Exactly what the distro's Custom Command Menu entry runs. + observe_launch menu bash -c "${BIN}/chairlift-wrapper --ask-bluefin" + + # A real answer with tools: the goose CLI through llmman in the same + # profile, so the same extensions and hints. The ground truth is + # what systemd itself reports as failed. + { + echo "system:"; systemctl --failed --no-legend --plain 2>&1 + echo "user:"; systemctl --user --failed --no-legend --plain 2>&1 + } >"${G}/failed-units-truth.txt" + cat >"${G}/extract.py" <<'PYX' + import json, os, re, sqlite3, sys + root, out, seen_files = sys.argv[1], sys.argv[2], sys.argv[3:] + seen_ids = set() + for f in seen_files: + try: + seen_ids.update(c["id"] for c in json.load(open(f))["tool_calls"]) + except (OSError, ValueError, KeyError): + pass + calls, responses, seen, assistant_texts = [], [], set(), [] + def walk(o): + if isinstance(o, dict): + kind = o.get("type") + if kind in ("toolRequest", "tool_request"): + call = o.get("toolCall") or o.get("tool_call") or {} + value = call.get("value", call) if isinstance(call, dict) else {} + key = (o.get("id"), json.dumps(value, sort_keys=True, default=str)) + if key not in seen and o.get("id") not in seen_ids: + seen.add(key) + calls.append({"id": o.get("id"), "name": value.get("name"), "arguments": value.get("arguments")}) + if kind in ("toolResponse", "tool_response") and o.get("id") not in seen_ids: + responses.append({"id": o.get("id"), "result": json.dumps(o.get("toolResult") or o.get("tool_result"), default=str)[:600]}) + for v in o.values(): + walk(v) + elif isinstance(o, list): + for v in o: + walk(v) + def texts(o): + if isinstance(o, dict): + if o.get("type") == "text" and isinstance(o.get("text"), str): + yield o["text"] + for v in o.values(): + yield from texts(v) + elif isinstance(o, list): + for v in o: + yield from texts(v) + db = os.path.join(root, "data", "sessions", "sessions.db") + con = sqlite3.connect(db) + tables = [r[0] for r in con.execute("select name from sqlite_master where type='table'")] + for table in tables: + cur = con.execute(f'select * from "{table}"') + columns = [d[0] for d in cur.description] + for row in cur: + record = dict(zip(columns, row)) + for cell in row: + if isinstance(cell, str) and re.search(r"tool_?[Rr]e(quest|sponse)", cell): + try: + walk(json.loads(cell)) + except ValueError: + pass + if table == "messages" and record.get("role") == "assistant": + for cell in row: + if isinstance(cell, str) and cell.startswith(("[", "{")): + try: + found = [t for t in texts(json.loads(cell)) if t.strip()] + except ValueError: + continue + if found: + assistant_texts.append("\n".join(found)) + names = [c["name"] or "" for c in calls] + json.dump({"tables": tables, "tool_calls": calls, "tool_responses": responses, + "final_text": assistant_texts[-1] if assistant_texts else "", + "linux_tools_called": any(n.startswith("linux-tools__") for n in names), + "search_knowledge_called": "bluefin-knowledge__search_knowledge" in names}, + open(out, "w"), indent=1, default=str) + PYX + PROMPT="Use the linux-tools extension to list failed systemd units on this machine, then use search_knowledge to search the Project Bluefin knowledge base for 'bootc update'. Report both results." + SEEN=() + for n in 1 2 3; do + if [ "${n}" -gt 1 ]; then + [ "$(cat "${LANE}/answer-quality")" = "true" ] || break + # Answer quality: stop at the first model after the smallest + # that names every failed unit systemd reports. + if [ "${n}" -gt 2 ] && grep -q '^quality=correct' "${G}/answer-$((n - 1)).txt" 2>/dev/null; then + break + fi + model_step "${n}" || break + fi + CRC=0 + env GOOSE_PATH_ROOT="${PROFILE}/goose" XDG_CONFIG_HOME="${PROFILE}/desktop" \ + PATH="${BIN}:/usr/bin:/bin" \ + timeout 900 "${LLMMAN}" launch goose --model bluefin-active -- run --text "${PROMPT}" \ + >"${G}/cli-${n}.log" 2>&1 "${G}/cli-${n}-rc.txt" + python3 "${G}/extract.py" "${PROFILE}/goose" "${G}/tool-calls-${n}.json" "${SEEN[@]}" >"${G}/extract-${n}.log" 2>&1 || true + SEEN+=("${G}/tool-calls-${n}.json") + python3 - "${G}/tool-calls-${n}.json" "${G}/failed-units-truth.txt" >"${G}/answer-${n}.txt" <<'PYQ' + import json, re, sys + result = json.load(open(sys.argv[1])) + truth = [line.split()[0] for line in open(sys.argv[2]).read().splitlines() + if line.strip() and not line.endswith(":") and not line.startswith(("system:", "user:"))] + text = result.get("final_text", "") + if truth: + correct = all(unit in text for unit in truth) + else: + correct = bool(re.search(r"\b(no|zero|0|none)\b[^.\n]{0,40}\bfail", text, re.I)) or "no failed" in text.lower() + print("quality=" + ("correct" if correct else "incorrect")) + print("truth=" + ",".join(truth)) + print("final_text=" + json.dumps(text)) + PYQ + done + + cp "${PROFILE}/goose/config/config.yaml" "${G}/config.after.yaml" 2>/dev/null + "${LANE}/bin/troubleshootcheck" enabled "${PROFILE}/goose/config/config.yaml" >"${G}/enabled-extensions.txt" 2>&1 + find "${PROFILE}" -maxdepth 4 \( -name 'Cache*' -o -name 'GPUCache' \) -prune -o -print | sort >"${G}/profile-tree.txt" + for d in "${HOME}/.config/goose" "${HOME}/.config/Goose" "${HOME}/.local/share/goose" "${HOME}/.local/state/goose"; do + if [ -e "${d}" ]; then echo "PRESENT ${d}"; find "${d}" -maxdepth 3 | head -50; else echo "ABSENT ${d}"; fi + done >"${G}/home-goose-paths.txt" + echo done >"${G}/rc.txt" + exit 0 + RUNNER + chmod 0755 /workspace/lane/run-chairlift.sh + + echo "Running ChairLift's behave suite in the nested Wayland session..." >&2 + QECORE_RC=0 + # The user's name, not 1000:1000, so supplementary groups survive. + podman exec \ + --user bluefin-test \ + --env USER=bluefin-test \ + --env HOME=/home/bluefin-test \ + --env XDG_RUNTIME_DIR=/run/user/1000 \ + --env DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + --env AT_SPI_BUS_ADDRESS=unix:path=/run/user/1000/bus \ + --env XDG_SESSION_TYPE=wayland \ + --env XDG_SESSION_DESKTOP=gnome \ + --env LIBGL_ALWAYS_SOFTWARE=1 \ + --env GALLIUM_DRIVER=llvmpipe \ + "${TARGET_NAME}" \ + qecore-headless \ + --session-type wayland \ + --session-desktop gnome \ + /workspace/lane/run-chairlift.sh || QECORE_RC=$? + echo "qecore-headless exit ${QECORE_RC}" >&2 + + # ---- verdicts ------------------------------------------------------ + BEHAVE_RC=$(cat /tmp/results/atspi/behave-rc.txt 2>/dev/null || printf '%s' "${QECORE_RC}") + SUITE_RC=0 + python3 - <<'PY' || SUITE_RC=$? + import json, os, re, sys + out = "/tmp/results" + path = os.path.join(out, "atspi", "results.json") + if not os.path.exists(path): + open(os.path.join(out, "result-summary.txt"), "w").write("Execution failed: no behave results") + sys.exit(1) + data = json.load(open(path, encoding="utf-8")) + counts = {"passed": 0, "failed": 0, "skipped": 0, "other": 0} + failed = [] + per_feature = {} + safe = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 .,:;()/_+'\"—-]{0,159}$") + for feature in data: + fc = per_feature.setdefault(feature.get("name", "?"), {"passed": 0, "failed": 0, "skipped": 0, "other": 0}) + for scenario in feature.get("elements", []): + if scenario.get("type") == "background": + continue + status = scenario.get("status", "other") + key = status if status in counts else "other" + counts[key] += 1 + fc[key] += 1 + if status == "failed" and safe.fullmatch(scenario.get("name", "")): + failed.append(scenario["name"]) + executed = counts["passed"] + counts["failed"] + counts["other"] + summary = f"{counts['passed']}/{executed} executed scenarios passed ({counts['failed']} failed, {counts['skipped']} skipped, {counts['other']} other)" + open(os.path.join(out, "result-summary.txt"), "w").write(summary) + json.dump(failed[:20], open(os.path.join(out, "failed-scenarios.json"), "w"), separators=(",", ":")) + json.dump({"counts": counts, "features": per_feature, "failed": failed}, + open(os.path.join(out, "scenario-counts.json"), "w"), indent=1) + print(summary) + for name, fc in per_feature.items(): + print(f" {name}: {fc}") + if executed == 0: + print("ERROR: 0 scenarios executed; a run with no executed tests cannot pass", file=sys.stderr) + sys.exit(1) + PY + + # Failed scenarios: the accessibility tree and the log tail, since + # Argo logs are the only view of a container lane. + for dir in /tmp/results/atspi/scenarios/*/; do + [ -f "${dir}tree.txt" ] || continue + echo "=== failed scenario: $(basename "${dir}") ===" >&2 + ls -l "${dir}" >&2 + head -n 120 "${dir}tree.txt" >&2 + tail -n 40 "${dir}chairlift.log" >&2 2>/dev/null || true + done + + # ---- walkthrough screenshots ------------------------------------ + # ChairLift's own capture (test/e2e/capture_walkthrough.sh) under its + # own wayland_session.sh: headless Mutter, native Wayland, frames from + # Mutter's ScreenCast, 900x700 as walkthrough_test.go sizes them. It + # runs on a private bus and runtime directory, beside the GDM session. + SHOTS_RC=0 + mapfile -t PAGE_NAMES < <(python3 -c 'import json; [print(i["name"]) for i in json.load(open("/workspace/lane/inventory/navigation.json"))]') + podman exec -i --user bluefin-test "${TARGET_NAME}" env -i \ + HOME=/home/bluefin-test PATH=/usr/bin:/bin LANG=C.UTF-8 \ + CHAIRLIFT_WAYLAND_SIZE=900x700 \ + CHAIRLIFT_SCHEMA_DIR=/home/bluefin-test/chairlift-schemas \ + CHAIRLIFT_ATSPI_PYTHON=/home/bluefin-test/atspi-venv/bin/python \ + bash -s -- "${PAGE_NAMES[@]}" <<'SHOTS' >/tmp/results/screenshots.log 2>&1 || SHOTS_RC=$? + set -euo pipefail + rm -rf /tmp/shots && mkdir -p /tmp/shots/bin /tmp/shots/runtime /tmp/results/screenshots-raw + chmod 0700 /tmp/shots/runtime + install -m 0755 /workspace/chairlift/build/e2e/chairlift /tmp/shots/bin/chairlift + export XDG_RUNTIME_DIR=/tmp/shots/runtime + timeout 600 dbus-run-session -- /workspace/chairlift/test/e2e/wayland_session.sh \ + /workspace/chairlift/test/e2e/capture_walkthrough.sh /tmp/shots/bin/chairlift /tmp/results/screenshots-raw "$@" + SHOTS + echo "${SHOTS_RC}" >/tmp/results/screenshots-rc.txt + # Only the PNGs are deliverable; the capture byproducts stay in -raw. + mkdir -p /tmp/results/screenshots + find /tmp/results/screenshots-raw -maxdepth 1 -name '[0-9]-*.png' -exec cp {} /tmp/results/screenshots/ \; + ls -l /tmp/results/screenshots >&2 + cp /workspace/lane/make-ci.log /workspace/lane/make-ci-rc /tmp/results/ 2>/dev/null || true + + GOOSE_RC=0 + G=/tmp/results/goose + GOOSE_STATE=$(cat "${G}/rc.txt" 2>/dev/null || echo not-run) + if [[ "${GOOSE_STATE}" == "skipped" ]]; then + echo '{"status":"skipped"}' >"${G}/verdict.json" + else + echo "=== Engine (${GOOSE_STATE}) ===" >&2 + for f in setup.log setup-rc.txt brew-installed.txt tap-info.json trust-state.txt polkit-session.txt \ + agent-mode.log agent-mode-rc.txt flatpaks.txt model-1.log model-1-rc.txt node-1.json llmman-ls-1.txt \ + command-env.txt shell-env.txt \ + askbluefin.log askbluefin-lock.txt askbluefin-environ.txt askbluefin-cmdline.txt askbluefin-processes.txt \ + menu.log menu-lock.txt menu-environ.txt menu-cmdline.txt menu-processes.txt relaunch-state.txt relaunch.log \ + failed-units-truth.txt \ + model-2.log model-2-rc.txt model-3.log model-3-rc.txt \ + cli-1-rc.txt cli-1.log answer-1.txt cli-2-rc.txt cli-2.log answer-2.txt cli-3-rc.txt cli-3.log answer-3.txt \ + enabled-extensions.txt home-goose-paths.txt; do + echo "--- ${f}" >&2 + cut -c1-600 "${G}/${f}" >&2 2>/dev/null || true + done + GOOSE_STATE="${GOOSE_STATE}" python3 - <<'PY' || GOOSE_RC=$? + import json, os, re, sys + g = "/tmp/results/goose" + def read(name): + try: + return open(os.path.join(g, name), encoding="utf-8", errors="replace").read() + except OSError: + return "" + def kv(name): + return dict(line.split("=", 1) for line in read(name).splitlines() if "=" in line) + state = os.environ["GOOSE_STATE"] + enabled = sorted(read("enabled-extensions.txt").split()) + paths = read("home-goose-paths.txt") + tree = read("profile-tree.txt").splitlines() + profile = "/home/bluefin-test/.local/share/chairlift/troubleshooting" + setup = kv("setup.log") + command_path = kv("command-env.txt").get("PATH", "") + brew_bin = os.path.dirname(read("brew-path.txt").strip()) + agent = kv("agent-mode.log") + model = kv("model-1.log") + try: + tap_info = json.loads(read("tap-info.json")) + tap_trusted = tap_info[0].get("trusted") if tap_info else None + except (ValueError, IndexError, AttributeError): + tap_trusted = None + try: + tools = json.loads(read("tool-calls-1.json")) + except ValueError: + tools = {} + + def launch_checks(tag): + lock = kv(f"{tag}-lock.txt") + env = kv(f"{tag}-environ.txt") + log = read(f"{tag}.log") + return { + f"{tag}_launcher_exit_zero": lock.get("launcher_exit") == "0", + f"{tag}_goose_running": lock.get("alive") == "yes", + f"{tag}_no_failure_logged": not re.search( + r"app: (launch goose desktop failed|goose desktop exited with error)|^Error:|^\[llmman\].*refus", + log, re.M), + f"{tag}_env_goose_path_root": env.get("GOOSE_PATH_ROOT") == profile + "/goose", + f"{tag}_env_xdg_config_home": env.get("XDG_CONFIG_HOME") == profile + "/desktop", + f"{tag}_env_goose_provider_openai": env.get("GOOSE_PROVIDER") == "openai", + f"{tag}_env_openai_host_agent_mode": "127.0.0.1:17434" in env.get("OPENAI_HOST", ""), + f"{tag}_env_goose_model_set": bool(env.get("GOOSE_MODEL")), + f"{tag}_lock_in_profile": lock.get("profile_lock", "ABSENT") != "ABSENT", + f"{tag}_no_lock_in_home_config": lock.get("home_lock", "ABSENT") == "ABSENT", + }, lock, env + + ask_checks, ask_lock, ask_env = launch_checks("askbluefin") + relaunch = kv("relaunch-state.txt") + def shell_json(value): + # Shell.Eval returns the stringified result as a JS string, inside + # a GVariant string: (true, '"{\\"windows\\":1,…}"'). + match = re.search(r"\(true, '(.*)'\)\s*$", value or "") + if not match: + return {} + try: + decoded = json.loads(match.group(1).replace("\\\\", "\\")) + return json.loads(decoded) if isinstance(decoded, str) else decoded + except ValueError: + return {} + windows_after = shell_json(relaunch.get("windows_after")) + relaunch_checks = { + "relaunch_while_running_exit_zero": relaunch.get("relaunch_exit") == "0", + "relaunch_no_second_goose": bool(relaunch.get("mains_before")) + and relaunch.get("mains_after") == relaunch.get("mains_before"), + "relaunch_goose_window_exists": windows_after.get("windows", 0) >= 1, + "relaunch_goose_window_raised": windows_after.get("minimized", 1) == 0 and windows_after.get("focused") is True, + } + menu_checks, menu_lock, menu_env = launch_checks("menu") + checks = { + "setup_ran_without_brew_on_path": setup.get("PATH") == "/usr/bin:/bin", + "chairlift_setup_exit_zero": read("setup-rc.txt").strip() == "0", + "goose_desktop_resolves_after_setup": setup.get("desktop") == os.path.join(brew_bin, "goose-desktop"), + "agent_mode_enabled_on_stock_host": read("agent-mode-rc.txt").strip() == "0" and agent.get("healthy") == "true", + "llmman_formula_trusted": "llmmanorg/tap/llmman" in read("trust-state.txt"), + "llmmanorg_tap_not_trusted": tap_trusted is False, + "pullmodel_succeeded_and_alias_set": read("model-1-rc.txt").strip() == "0" + and bool(model.get("ref")) and model.get("active", "") in (model["ref"], "hf.co/" + model["ref"]), + "command_env_path_has_brew_bin_first": command_path.split(":")[0] == brew_bin, + **ask_checks, + **menu_checks, + **relaunch_checks, + "menu_same_goose_env_as_ask_bluefin": bool(ask_env) and all( + ask_env.get(k) == menu_env.get(k) + for k in ("GOOSE_PATH_ROOT", "XDG_CONFIG_HOME", "GOOSE_PROVIDER", "OPENAI_HOST", "GOOSE_MODEL")), + "cli_exit_zero": read("cli-1-rc.txt").strip() == "0", + "linux_tools_tool_called": tools.get("linux_tools_called") is True, + "search_knowledge_tool_called": tools.get("search_knowledge_called") is True, + "only_linux_tools_and_bluefin_knowledge_enabled": enabled == ["bluefin-knowledge", "linux-tools"], + "goose_data_or_state_under_path_root": any(p.startswith(profile + "/goose/data/") or p.startswith(profile + "/goose/state/") for p in tree), + "nothing_in_home_config_goose": "ABSENT /home/bluefin-test/.config/goose" in paths, + } + answers = {} + for n in ("1", "2", "3"): + if read(f"cli-{n}-rc.txt"): + answers[n] = {"model": kv(f"model-{n}.log").get("ref") if n != "1" else model.get("ref"), + "pull": {k: kv(f"model-{n}.log").get(k) for k in ("size_bytes", "pull_seconds")}, + "cli_rc": read(f"cli-{n}-rc.txt").strip(), + "quality": kv(f"answer-{n}.txt").get("quality"), + "tool_calls": [c.get("name") for c in json.loads(read(f"tool-calls-{n}.json") or "{}").get("tool_calls", [])]} + verdict = {"status": "pass" if all(checks.values()) else "fail", "state": state, "checks": checks, + "model": model, "askbluefin": {"lock": ask_lock, "env": ask_env}, + "menu": {"lock": menu_lock, "env": menu_env}, "enabled": enabled, "answers": answers, + "relaunch": {"state": relaunch, "windows_before": shell_json(relaunch.get("windows_before")), + "windows_after": windows_after}} + json.dump(verdict, open(os.path.join(g, "verdict.json"), "w"), indent=1) + print(json.dumps(verdict, indent=1)) + sys.exit(0 if verdict["status"] == "pass" else 1) + PY + fi + + CI_RC=$(cat /workspace/lane/make-ci-rc 2>/dev/null || echo missing) + SHOTS_OK=0 + { [ "${SHOTS_RC}" = 0 ] && grep -q 'walkthrough complete' /tmp/results/screenshots.log \ + && grep -q 'views: reset group built' /tmp/results/screenshots-raw/chairlift.log \ + && [ "$(find /tmp/results/screenshots -name '*.png' | wc -l)" -eq $(( ${#PAGE_NAMES[@]} + 1 )) ]; } || SHOTS_OK=1 + echo "make ci exit ${CI_RC}; behave exit ${BEHAVE_RC}; suite verdict ${SUITE_RC}; engine verdict ${GOOSE_RC}; screenshots ${SHOTS_OK}" | tee /tmp/results/lane-summary.txt >&2 + if [[ "${CI_RC}" != "0" || "${BEHAVE_RC}" != "0" || "${SUITE_RC}" != "0" || "${GOOSE_RC}" != "0" || "${SHOTS_OK}" != "0" ]]; then + exit 1 + fi + volumes: + - name: workspace + emptyDir: {} + - name: results + emptyDir: {} + - name: containers-storage + emptyDir: {} + - name: containers-runroot + emptyDir: {} + - name: udev + hostPath: + path: /run/udev + type: Directory + - name: results-store + hostPath: + path: /var/mnt/ghost-data/local-path + type: Directory + - name: pip-cache + hostPath: + path: /var/cache/bluefin-qa-pip + type: DirectoryOrCreate diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index c6c0a0d1ad..afa3ad30f1 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -14,6 +14,7 @@ template add or rename. - [dakota-build-pipeline](#dakota-build-pipeline) - [zot-candidate-lifecycle](#zot-candidate-lifecycle) - [bluefin-server-build-pipeline](#bluefin-server-build-pipeline) + - [run-chairlift-wayland-tests](#run-chairlift-wayland-tests) - [KubeStellar Workflows](#kubestellar-workflows) - [Supporting Templates](#supporting-templates) - [Distributed Build/RE Grid](#distributed-buildre-grid) @@ -126,6 +127,97 @@ before retrying. and execution are identical to dakota. - **Cache policy:** CAS (`cache.storage-service`) and action cache use the shared Buildbarn frontend (`frontend.buildbarn.svc.cluster.local:8980`); artifacts and sources are indexed in `bb-remote-asset` (`:8984`) and pushed there, with upstream caches as read-only fallbacks. Source caches override the project entries. See `manifests/buildstream-remote-cache-config.yaml`. +### run-chairlift-wayland-tests +- **Purpose:** ChairLift's behave + dogtail AT-SPI suite + (`test/e2e/features`) in a live GNOME **Wayland** session, plus an + Enhanced Troubleshooting Goose-profile isolation check on x86_64. +- **Parameters:** `chairlift-repo`, `chairlift-branch`, `image` + (`ghcr.io/projectbluefin/dakota`), `image-tag` (`testing`), `behave-tags` + (empty = every scenario), `goose-isolation-check` (`true`). +- **Shape:** a `golang:1.26` init container clones the branch and runs + ChairLift's `make ci` on the untouched checkout. It fetches golangci-lint + v2.12.2, the version CI pins, by checksum, and records the result, which + the verdict gates on. It then runs `make build-e2e schemas`, stages + `chairlift` and `chairlift-wrapper` the way the Homebrew cask installs + them, and builds two lane tools inside the module: `navjson` (the + page/shortcut inventory the Go gate normally passes) and + `troubleshootcheck`. `troubleshootcheck` drives the branch's + `internal/troubleshoot` and `internal/aistack`: Setup, Agent Mode, the + model alias path, the PATH `Command` hands the launch, and a config + reader. + The runner then boots `run-container-tests`' nested systemd/GDM target: + headless GNOME Shell drop-in, linger, `lastchg`, and the resolver bind. + `/usr/share/chairlift` is masked and `/proc/cmdline` blanked, as the fixture + harness does. Behave runs from a hash-pinned venv under + `qecore-headless --session-type wayland`, with `GDK_BACKEND=wayland`, an + absolute `WAYLAND_DISPLAY`, and `DISPLAY` unset. Keys go through dogtail's + Mutter RemoteDesktop backend. Xvfb is never used. +- **Screenshots:** ChairLift's own `capture_walkthrough.sh` runs under its + `wayland_session.sh` (headless Mutter at 900x700, frames from Mutter's + ScreenCast) inside the target, on a private bus beside the GDM session. + Only the page PNGs are kept, in `screenshots/`; the byproducts stay in + `screenshots-raw/`. +- **Wayland traps:** qecore hands the script gnome-session's environment, + which has no `WAYLAND_DISPLAY`; the script reads it from + `systemctl --user show-environment`. Mutter's focus-stealing prevention + leaves each new window unfocused, so ChairLift's `environment.py` + activates it through `Shell.Eval` (needs `--unsafe-mode`). Dakota's + `brew-preinstall.service` and brew timers are masked so they cannot hold + Homebrew's locks during the Goose check. +- **Goose check (end to end, ChairLift's own code throughout):** + 1. `troubleshoot.Setup` installs the tap, `linux-mcp-server`, `cpio`, and + the `goose-linux` cask with `PATH=/usr/bin:/bin` and no + `brew shellenv`, as a direct `chairlift` launch would. The cask's bare + `rpm2cpio | cpio` preflight exits 127 unless ChairLift puts Homebrew's + bin on brew's PATH. + 2. `aistack.Enable` turns Agent Mode on, on the stock host. The check + records that `brew trust` covers the `llmman` formula and not + `llmmanorg/tap`. + 3. The smallest Qwen3 is resolved through aistack's live resolver, pulled + with `aistack.PullModel` (which must succeed on its own), and set as the + `bluefin-active` alias. llmman reports the alias as the + registry-qualified `hf.co/`. The lane also records `/llmman/node` + and `llmman ls`. + 4. Two launches run with GNOME Shell's own environment and a PATH without + Homebrew, in an app.slice scope. The first is a cold + `chairlift --ask-bluefin`. The second is the distro menu's exact + command, `bash -c '…/chairlift-wrapper --ask-bluefin'`. Each must + exit 0 with Goose running through llmman. The lane reads the + environment Goose hands its `goose serve` backend (`GOOSE_PATH_ROOT`, + `XDG_CONFIG_HOME`, `GOOSE_PROVIDER=openai`, `OPENAI_HOST` on + `127.0.0.1:17434`, `GOOSE_MODEL`); the two launches must agree on it. + It also checks that the lock is in the profile and takes a screenshot. + 5. `llmman launch goose -- run --text …`, run in the same profile, must + call a `linux-tools__*` tool and `bluefin-knowledge__search_knowledge`, + read back from `data/sessions/sessions.db`. The answer is compared + with `systemctl --failed` for 0.6B, 1.7B, and then 4B, and recorded. + Answer quality does not gate the run. + + Throughout, the profile's lock, data, state, and config must stay inside + it, and only `linux-tools` and `bluefin-knowledge` may be enabled. +- **ChairLift runs as an app unit:** every ChairLift step runs through + `systemd-run --user --slice=app.slice`, the way GNOME starts an app. The + script itself runs in podman's exec cgroup, outside any logind session, + where polkit answers `auth_admin` to the system Flatpak install Agent Mode + does; inside an app unit, an active `wheel` member gets `yes`. The test + user joins `wheel`, as Bluefin's first user does. `TasksMax` is lifted on + the user slice and on the units, because brew's download threads hit the + container-sized default. +- **Verdicts:** the run fails if any of these happens: + - `make ci` fails; + - behave fails, or executes 0 scenarios; + - any engine check fails; + - the screenshot capture does not produce every page and the + reset-group marker. + + Outputs `result`, `failed-scenarios`, and + `goose-isolation` (JSON). Everything under `/tmp/results` (behave log, + JUnit, per-scenario `chairlift.log`, `tree.txt`, `screen.png`; Goose + `setup.log`, `command-env.txt`, lock evidence, config, screenshot) is persisted to + ghost's result store under `/chairlift-wayland/`. +- **Submit:** not deployed until merged; until then submit the template + file as a Workflow (`kind: Workflow`, `generateName`) with `argo submit`. + ## KubeStellar Workflows KubeStellar installation and upgrades are owned by the `kubestellar-applications` @@ -248,5 +340,7 @@ Pod resource requests/limits used by workflow steps: | Template / container | CPU req/limit | Memory req/limit | | --- | --- | --- | | `run-container-tests` | 2 / 4 | 4Gi / 8Gi | +| `run-chairlift-wayland-tests` `main` | 2 / 4 | 4Gi / 8Gi | +| `run-chairlift-wayland-tests` `build-chairlift` | 2 / 4 | 2Gi / 4Gi | | `bst-build-re` `main` | 2 / 4 | 4Gi / 8Gi | | `bst-build-re` `publish` | 1 / 16 | 256Mi / 2Gi | diff --git a/docs/skills/argo-workflows/patterns.md b/docs/skills/argo-workflows/patterns.md index 076f517044..3970c788b8 100644 --- a/docs/skills/argo-workflows/patterns.md +++ b/docs/skills/argo-workflows/patterns.md @@ -137,6 +137,45 @@ NVIDIA image builds are disabled for this clean distributed-build path. and `image-poll-dakota` (not suspended) drives it on every new `dakota:testing` digest. +### Driving a GTK app through AT-SPI in the nested Wayland session + +`run-chairlift-wayland-tests` runs an application's own behave suite inside +`run-container-tests`' nested GNOME session. Three traps, each observed on +ghost: + +- **qecore's script environment has no `WAYLAND_DISPLAY`.** qecore-headless + copies gnome-session's environment, which predates the compositor, and sets + `DISPLAY=:0` (Xwayland). Read `WAYLAND_DISPLAY` from + `systemctl --user show-environment`, check the socket, set + `GDK_BACKEND=wayland`, and unset `DISPLAY` so nothing falls back to X. +- **New windows are not focused.** Mutter's focus-stealing prevention leaves a + window mapped without an activation token unfocused (`focus_window` is + null), so every synthesized key reaches nothing. Activate it with + `Main.activateWindow` over `Shell.Eval` (the headless drop-in's + `--unsafe-mode` allows it) and wait for `global.display.focus_window`. +- **The first ponytail key is dropped** while gnome-ponytail-daemon's + remote-desktop session starts. Spend it on a bare Shift once per run. + +Screenshots come from `org.gnome.Shell.Screenshot.Screenshot`, which +`--unsafe-mode` also opens to the test user; there is no root window for `xwd`. + +Two more traps, from driving the app's own privileged-adjacent paths: + +- **`podman exec` is not in the user's session.** A process started with + `podman exec` sits in the target's exec cgroup, so logind assigns it to no + session and polkit answers `auth_admin` to actions it allows an active + local `wheel` member without a prompt (a system Flatpak install, for one). + Run app-side steps the way GNOME starts an app: + `systemd-run --user --wait --pipe --slice=app.slice`. `pkcheck` from each + context is the evidence. To hand `pkcheck` a PID, read it from + `/proc/self/stat`. `systemd-run` expands `${VAR}` in its command line + itself, and in the qecore-launched script `$$` reached `pkcheck` as a bare + `$` (the cause was not isolated). +- **The user slice's `TasksMax` is container-sized.** Inside a unit, brew's + download threads failed with "can't create Thread: Resource temporarily + unavailable". Lift `TasksMax` on `user-1000.slice`, `app.slice`, and the + unit itself. + ## Concurrency and scheduling ### VM concurrency: k8s native scheduling, no semaphores diff --git a/tests/unit/test_run_chairlift_wayland_tests.py b/tests/unit/test_run_chairlift_wayland_tests.py new file mode 100644 index 0000000000..ebf79936ea --- /dev/null +++ b/tests/unit/test_run_chairlift_wayland_tests.py @@ -0,0 +1,209 @@ +"""Static contract for run-chairlift-wayland-tests (ChairLift's live Wayland lane).""" + +import re +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATE = ROOT / "argo/workflow-templates/run-chairlift-wayland-tests.yaml" + + +def _doc(): + return yaml.safe_load(TEMPLATE.read_text(encoding="utf-8")) + + +def _template(): + return next(t for t in _doc()["spec"]["templates"] if t["name"] == "run-chairlift-wayland-tests") + + +def _runner(): + return _template()["script"]["source"] + + +def _heredoc(source, tag): + match = re.search(rf"<<'{tag}'[^\n]*\n(.*?)\n\s*{tag}\n", source, re.S) + assert match, f"heredoc {tag} not found" + return match.group(1) + + +def _bash_parses(script): + result = subprocess.run(["bash", "-n"], input=script, text=True, capture_output=True) + assert result.returncode == 0, result.stderr + + +def test_parameters_are_the_lane_contract(): + names = [p["name"] for p in _doc()["spec"]["arguments"]["parameters"]] + assert names == [ + "chairlift-repo", + "chairlift-branch", + "image", + "image-tag", + "behave-tags", + "goose-isolation-check", + "answer-quality", + ] + defaults = {p["name"]: p["value"] for p in _doc()["spec"]["arguments"]["parameters"]} + assert defaults["image"] == "ghcr.io/projectbluefin/dakota" + assert defaults["image-tag"] == "testing" + + +def test_runs_on_ghost_behind_the_container_qa_semaphore(): + template = _template() + assert template["nodeSelector"]["kubernetes.io/hostname"] == "ghost" + keys = [s["configMapKeyRef"]["key"] for s in template["synchronization"]["semaphores"]] + assert keys == ["ghost-container-qa"] + assert template["activeDeadlineSeconds"] <= 7200 + + +def test_builds_the_requested_branch_with_the_e2e_target(): + build = _template()["initContainers"][0] + script = build["args"][0] + assert 'git clone --depth 1 --branch "${CHAIRLIFT_BRANCH}" "${CHAIRLIFT_REPO}"' in script + assert "CGO_ENABLED=0 make build-e2e schemas" in script + # ChairLift's whole CI gate runs on the untouched checkout, with the + # golangci-lint CI pins, verified by checksum, before any lane tool exists. + assert script.index("make ci >") < script.index("build/lane/navjson") + assert 'echo "${LINT_SHA256} /tmp/${LINT_TGZ}" | sha256sum -c -' in script + assert "troubleshoot.ProfileAt(os.Args[2])" in script + # The install goes through ChairLift's own Setup, not hand-written brew. + assert "troubleshoot.Setup(troubleshoot.Detect()," in script + assert "navigation.Items()" in script + # Installed the way the Homebrew cask lays ChairLift out. + assert "install -m 0755 data/chairlift-wrapper.sh /workspace/lane/install/chairlift-wrapper" in script + _bash_parses(script) + + +def test_boots_the_run_container_tests_wayland_target(): + source = _runner() + setup = _heredoc(source, "NESTED_SETUP") + assert "--systemd=always" in source and "--privileged" in source + # Rule 12: headless shell, never DRM master, Shell.Eval kept. + assert "--mode=%i --unsafe-mode --headless --virtual-monitor 1920x1080" in setup + assert "loginctl enable-linger bluefin-test" in setup + assert 'chage -d "$(date +%Y-%m-%d)" bluefin-test' in setup + assert "--volume /etc/resolv.conf:/etc/resolv.conf:ro" in source + assert "qecore-headless \\\n --session-type wayland" in source + assert "systemctl --global mask brew-preinstall.service" in setup + _bash_parses(source) + _bash_parses(setup) + + +def test_wayland_only_never_x11(): + text = TEMPLATE.read_text(encoding="utf-8") + for banned in ("Xvfb", "xvfb", "GDK_BACKEND=x11", "xwd", "DISPLAY=:"): + assert banned not in text, banned + runner = _heredoc(_runner(), "RUNNER") + assert "GDK_BACKEND=wayland" in runner + assert "unset DISPLAY XAUTHORITY" in runner + # Goose's window comes up exactly as ChairLift launches it: no Electron + # platform hint, and no DISPLAY to fall back to (unset above). + assert "ELECTRON_OZONE_PLATFORM_HINT" not in runner + assert "org.gnome.Shell.Screenshot" in runner + _bash_parses(runner) + + +def test_no_shell_modal_holds_the_keyboard(): + source = _runner() + setup = _heredoc(source, "NESTED_SETUP") + assert "gsettings set org.gnome.shell welcome-dialog-last-shown-version" in setup + runner = _heredoc(source, "RUNNER") + assert "'Main.modalCount'" in runner + + +def test_suite_isolation_matches_the_fixture_harness(): + source = _runner() + runner = _heredoc(source, "RUNNER") + # A shipped config.yml outranks every fixture; the suite refuses it. + assert "--tmpfs /usr/share/chairlift:ro,notmpcopyup" in source + assert "destination=/proc/cmdline,ro" in source + for setting in ("GSETTINGS_BACKEND=memory", "GDK_DEBUG=no-portals", "GTK_A11Y=atspi", "CHAIRLIFT_REQUIRE_ATSPI=1"): + assert setting in runner, setting + assert "--require-hashes -r /workspace/chairlift/test/e2e/requirements-atspi.txt" in source + + +def test_fails_closed_and_cleans_up(): + source = _runner() + assert "trap on_exit EXIT" in source + assert "trap 'exit 143' TERM" in source + assert "trap 'exit 130' INT" in source + assert 'podman rm --force "${TARGET_NAME}"' in source + assert "0 scenarios executed" in source + assert "sys.exit(1)" in source + + +def test_engine_check_runs_chairlifts_own_paths(): + runner = _heredoc(_runner(), "RUNNER") + # ChairLift installs the packages itself, with no Homebrew on PATH and + # no `brew shellenv`, the way a direct launch runs it. + assert "PATH=/usr/bin:/bin" in runner + assert '"${LANE}/bin/troubleshootcheck" setup' in runner + assert '"${LANE}/bin/troubleshootcheck" command-path' in runner + assert 'eval "$("${BREW}" shellenv)"' not in runner + assert '"${BREW}" install' not in runner + assert '"${LANE}/bin/troubleshootcheck" agent-mode' in runner + assert '"${LANE}/bin/troubleshootcheck" model' in runner + # No lane fallbacks: PullModel must succeed, and nothing is trusted or + # aliased on ChairLift's behalf. + for fallback in ("troubleshootcheck alias", "not found in daemon stored models", '"${BREW}" trust', "brew trust"): + assert fallback not in runner, fallback + build = _template()["initContainers"][0]["args"][0] + assert "/usr/bin/true" not in build + for call in ("aistack.Enable(ctx)", "aistack.WaitHealthy(", "aistack.PullModel(", "aistack.ConfigureActiveModel("): + assert call in build, call + # Goose is launched by ChairLift itself: a cold --ask-bluefin, and the + # distro menu's exact command, both with GNOME Shell's environment. + assert 'observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin' in runner + assert 'observe_launch menu bash -c "${BIN}/chairlift-wrapper --ask-bluefin"' in runner + assert 'env -i "${SHELL_ENV[@]}" PATH="${MENU_PATH}"' in runner + assert "grep -v linuxbrew" in runner + # Nothing fakes the provider: llmman hands Goose its environment. + for fake in ("GOOSE_PROVIDER=openai", "OPENAI_HOST=http://127.0.0.1:9", "OPENAI_API_KEY=x"): + assert fake not in runner, fake + assert 'launch goose --model bluefin-active -- run --text' in runner + assert "systemctl --failed --no-legend --plain" in runner + source = _runner() + for check in ( + "setup_ran_without_brew_on_path", + "chairlift_setup_exit_zero", + "goose_desktop_resolves_after_setup", + "agent_mode_enabled_on_stock_host", + "llmman_formula_trusted", + "llmmanorg_tap_not_trusted", + "pullmodel_succeeded_and_alias_set", + "command_env_path_has_brew_bin_first", + "_launcher_exit_zero", + "_env_goose_path_root", + "_env_openai_host_agent_mode", + "_lock_in_profile", + "menu_same_goose_env_as_ask_bluefin", + "linux_tools_tool_called", + "search_knowledge_tool_called", + "only_linux_tools_and_bluefin_knowledge_enabled", + "goose_data_or_state_under_path_root", + "nothing_in_home_config_goose", + ): + assert check in source, check + + +def test_screenshots_use_chairlifts_own_wayland_capture(): + source = _runner() + assert "/workspace/chairlift/test/e2e/wayland_session.sh" in source + assert "/workspace/chairlift/test/e2e/capture_walkthrough.sh" in source + assert "CHAIRLIFT_WAYLAND_SIZE=900x700" in source + assert "views: reset group built" in source + # Byproducts stay behind; only the page PNGs are deliverable. + assert "-name '[0-9]-*.png'" in source + assert 'CI_RC}" != "0"' in source + + +def test_relaunch_while_goose_runs_reuses_the_session(): + runner = _heredoc(_runner(), "RUNNER") + assert 'KEEP_GOOSE=1 observe_launch askbluefin "${BIN}/chairlift" --ask-bluefin' in runner + assert 'bash -c "${BIN}/chairlift-wrapper --ask-bluefin"' in runner + assert "mains_before=" in runner and "mains_after=" in runner + source = _runner() + for check in ("relaunch_while_running_exit_zero", "relaunch_no_second_goose", "relaunch_goose_window_exists", + "relaunch_goose_window_raised"): + assert check in source, check