wip: probe agent and ask failures on the 120 branch

.claude/skills/e2e-from-prs/SKILL.md

@@ -31,8 +31,7 @@ or code the e2e user cannot reach. Also skip when an existing test already asser
 
 ## 3. Write at most 3 tests
 
-1. Read `references/playwright.md` (next to this file), every file in `e2e/helpers/`, and the whole
-   target spec before you write anything.
+1. Read every file in `e2e/helpers/` and the whole target spec before you write anything.
 2. Pick the spec by area: `agent`, `ask`, `tree` (sections), `crop` (figures, pages), `publish`
    (wiki), `upload`, `parse` (import, sectionize, jobs). Never add to `smoke.spec.ts`. Create
    `e2e/specs/<area>.spec.ts` only if no area fits.
@@ -50,6 +49,7 @@ Rules:
 - Data you create is titled `` `${PREFIX} <area> W<n> ${Date.now()}` `` and deleted in `afterAll`
   (`deleteByPrefix`, `deleteTestProjects`, `deleteImport`). Never delete `[fixture]` records. A test
   that changes a fixture restores it in `afterEach` (see `restoreTree` in `tree.spec.ts`).
+- Make test data in code (`helpers/pdf.ts`, API calls); never add files.
 - Wait on database state with `waitForValue` or `waitFor` through `api`. Never wait on UI progress
   text, and never use `page.waitForTimeout`.
 - `@llm` tests: `test.describe.configure({ retries: 1 })`, assert records through `api`, never the

.github/actions/e2e-site/action.yml

@@ -95,7 +95,8 @@ runs:
       working-directory: /home/runner/frappe-bench
       run: |
         nohup bench serve --port 8000 --noreload > logs/e2e-web.log 2>&1 &
-        nohup bench worker > logs/e2e-worker.log 2>&1 &
+        # With CI set, frappe runs every job in test mode, which turns Local models off.
+        nohup env -u CI bench worker > logs/e2e-worker.log 2>&1 &
         nohup node apps/frappe/socketio.js > logs/e2e-socketio.log 2>&1 &
         for attempt in $(seq 1 60); do
           curl -sf http://wikify.localhost:8000/api/method/ping && exit 0

.github/workflows/agent-ci-probe.yml

@@ -0,0 +1,53 @@
+name: Agent CI probe
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/agent-ci-probe.yml
+      - wikify/agent_ci_probe.py
+
+permissions:
+  contents: read
+
+jobs:
+  probe:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+    services:
+      redis-cache:
+        image: redis:alpine
+        ports:
+          - 13000:6379
+      redis-queue:
+        image: redis:alpine
+        ports:
+          - 11000:6379
+      mariadb:
+        image: mariadb:11.8
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306:3306
+        options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ./.github/actions/e2e-site
+      - name: Direct turn
+        shell: bash
+        working-directory: /home/runner/frappe-bench
+        run: bench --site wikify.localhost execute wikify.agent_ci_probe.direct
+      - name: Ask
+        if: always()
+        shell: bash
+        working-directory: /home/runner/frappe-bench
+        run: bench --site wikify.localhost execute wikify.agent_ci_probe.ask
+      - name: Worker turn
+        if: always()
+        shell: bash
+        working-directory: /home/runner/frappe-bench
+        run: |
+          bench --site wikify.localhost execute wikify.agent_ci_probe.worker
+          echo "----- worker log tail"
+          tail -n 80 logs/e2e-worker.log

.github/workflows/e2e.yml

@@ -9,11 +9,15 @@ on:
       - .github/workflows/e2e.yml
       - .github/actions/e2e-site/**
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   playwright:
     name: Playwright
     runs-on: ubuntu-latest
-    timeout-minutes: 240
+    timeout-minutes: 300
     permissions:
       contents: read
       issues: write

frontend/e2e/.env.e2e.example

@@ -3,3 +3,4 @@ E2E_USER=e2e@example.com
 E2E_PASSWORD=
 E2E_API_KEY=
 E2E_API_SECRET=
+# E2E_FIXTURE_PDF=../docs/what-is-wikify.pdf  # small doc for local runs until the bench has the 6 h parse timeout

frontend/e2e/fixtures/README.md

@@ -0,0 +1 @@
+Adapted from *Anatomy and Physiology*, J. Gordon Betts et al., OpenStax (2013), pp. 1155–1252, CC BY 4.0, https://openstax.org/details/books/anatomy-and-physiology. Changes: page range extracted; three University of Michigan micrographs (CC BY-NC-SA) removed.

frontend/e2e/helpers/cleanup.ts

@@ -1,5 +1,6 @@
 import type { Api } from "./api";
 import { waitFor } from "./wait";
+import { renumberTree } from "./wikify";
 
 const BUSY = ["Queued", "Parsing", "Remediating", "Generating Wiki"];
 
@@ -51,17 +52,32 @@ export async function waitForIdle(
 	).catch((error) => console.warn(`cleanup: ${error.message}`));
 }
 
+// A REST delete runs NestedSet's table-wide lft shift, which reorders other documents' trees
+// (the [fixture] source among them). delete_section deletes raw and renumbers only this document.
+async function deleteSections(api: Api, sourceDocument: string): Promise<void> {
+	const filters = { source_document: sourceDocument, parent_source_section: ["is", "not set"] };
+	const roots = await api.getList("Source Section", { filters });
+	if (roots.length) await renumberTree(api, sourceDocument);
+	for (const root of roots) {
+		await api
+			.call("wikify.api.sections.delete_section", { name: root.name })
+			.catch((error) => console.warn(`cleanup: delete_section ${root.name}: ${error.message}`));
+	}
+	await deleteTree(api, "Source Section", { source_document: sourceDocument });
+}
+
 export async function deleteImport(api: Api, importName: string): Promise<void> {
 	await waitForIdle(api, importName);
 	const imp = await api.getValue("Wikify Import", importName, ["source_document"]);
 	const sourceDocument = imp?.source_document;
 	await deleteByFilter(api, "Import Log Entry", { import: importName });
 	if (sourceDocument) {
+		// Unlinked first: delete_section refuses a document whose import is Completed.
+		await api.setValue("Wikify Import", importName, { source_document: null });
 		await deleteByFilter(api, "Wikify Agent Session", { source_document: sourceDocument });
-		await deleteTree(api, "Source Section", { source_document: sourceDocument });
+		await deleteSections(api, sourceDocument);
 		await deleteByFilter(api, "Section Reference", { source_document: sourceDocument });
 		await deleteByFilter(api, "Source Page", { source_document: sourceDocument });
-		await api.setValue("Wikify Import", importName, { source_document: null });
 		await api.setValue("Source Document", sourceDocument, {
 			wiki_root_group: null,
 			wiki_space: null,

frontend/e2e/helpers/env.ts

@@ -21,4 +21,13 @@ export const PREFIX = "[test]";
 export const FIXTURE_PREFIX = "[fixture]";
 
 export const STATE_DIR = path.resolve(import.meta.dirname, "../.state");
-export const FIXTURE_PDF = path.resolve(import.meta.dirname, "../../../docs/what-is-wikify.pdf");
+export const WHAT_IS_WIKIFY_PDF = path.resolve(
+	import.meta.dirname,
+	"../../../docs/what-is-wikify.pdf",
+);
+export const FIXTURE_PDF_OVERRIDDEN = !!process.env.E2E_FIXTURE_PDF;
+export const FIXTURE_PDF = FIXTURE_PDF_OVERRIDDEN
+	? path.resolve(import.meta.dirname, "../..", process.env.E2E_FIXTURE_PDF!)
+	: path.resolve(import.meta.dirname, "../fixtures/openstax-anatomy-ch24-25.pdf");
+export const FIXTURE_PDF_SHA256 =
+	"62d650cf40e02faea4395dc2301891c9e5165f88630713d4d92b7f704d0d6fcf";

frontend/e2e/helpers/test.ts

@@ -4,11 +4,21 @@ import { test as base } from "@playwright/test";
 import { Api } from "./api";
 import { STATE_DIR } from "./env";
 
-export type ImportFixture = { import: string; sourceDocument: string; title: string };
+export type OutlineEntry = { name: string; title: string; page_start: number; page_end: number };
+
+export type FixtureOutline = { root: OutlineEntry | null; children: OutlineEntry[] };
+
+export type ImportFixture = {
+	import: string;
+	sourceDocument: string;
+	title: string;
+	outline: FixtureOutline;
+};
 
 export type Fixtures = {
 	project: string;
 	projectName: string;
+	source: ImportFixture;
 	review: ImportFixture;
 	agent: ImportFixture;
 };