From a2b5e99c58e6ea28b4979034682268f153f08cd3 Mon Sep 17 00:00:00 2001 From: cerberus-ab Date: Sun, 16 Aug 2026 23:43:41 +0200 Subject: [PATCH 01/15] Init ContainerTree --- .../src/agent/tools/AbstractCallableTool.ts | 4 +- .../AbstractDocumentTransformer.ts | 4 +- apps/backend/src/types/documents.ts | 4 +- apps/extension/src/core/constants.ts | 1 + .../page/components/Inspector/Inspector.tsx | 6 +- .../components/Metadata/PageMetadata.tsx | 2 +- apps/extension/src/page/hooks/usePage.ts | 1 + .../src/collector/ElementRegistry.ts | 21 ++ .../src/collector/PageTrailCollector.test.ts | 31 ++- .../src/collector/PageTrailCollector.ts | 248 ++++-------------- .../{extractors => context}/context.test.ts | 0 .../{extractors => context}/context.ts | 18 +- .../extractors/ContainerTree.test.ts | 195 ++++++++++++++ .../src/collector/extractors/ContainerTree.ts | 153 +++++++++++ .../src/collector/extractors/basics.test.ts | 30 +++ .../src/collector/extractors/basics.ts | 24 ++ .../src/collector/extractors/content.test.ts | 74 ++++++ .../src/collector/extractors/content.ts | 68 +++++ .../collector/extractors/interactive.test.ts | 103 ++++++++ .../src/collector/extractors/interactive.ts | 83 ++++++ .../extractors/primitive/heading.test.ts | 93 +++++++ .../collector/extractors/primitive/heading.ts | 27 ++ .../extractors/primitive/label.test.ts | 56 +++- .../collector/extractors/primitive/label.ts | 88 +++++-- .../extractors/primitive/role.test.ts | 14 +- .../collector/extractors/primitive/role.ts | 50 +++- .../extractors/primitive/selector.ts | 7 + .../src/collector/importance/interactive.ts | 4 +- .../src/collector/selectors.test.ts | 173 ++++++++++++ .../page-trail/src/collector/selectors.ts | 14 + packages/page-trail/src/semantic/format.ts | 4 +- packages/page-trail/src/types/pageTrail.ts | 102 ++++--- 32 files changed, 1418 insertions(+), 284 deletions(-) create mode 100644 packages/page-trail/src/collector/ElementRegistry.ts rename packages/page-trail/src/collector/{extractors => context}/context.test.ts (100%) rename packages/page-trail/src/collector/{extractors => context}/context.ts (88%) create mode 100644 packages/page-trail/src/collector/extractors/ContainerTree.test.ts create mode 100644 packages/page-trail/src/collector/extractors/ContainerTree.ts create mode 100644 packages/page-trail/src/collector/extractors/basics.test.ts create mode 100644 packages/page-trail/src/collector/extractors/basics.ts create mode 100644 packages/page-trail/src/collector/extractors/content.test.ts create mode 100644 packages/page-trail/src/collector/extractors/content.ts create mode 100644 packages/page-trail/src/collector/extractors/interactive.test.ts create mode 100644 packages/page-trail/src/collector/extractors/interactive.ts create mode 100644 packages/page-trail/src/collector/extractors/primitive/heading.test.ts create mode 100644 packages/page-trail/src/collector/extractors/primitive/heading.ts create mode 100644 packages/page-trail/src/collector/extractors/primitive/selector.ts create mode 100644 packages/page-trail/src/collector/selectors.test.ts create mode 100644 packages/page-trail/src/collector/selectors.ts diff --git a/apps/backend/src/agent/tools/AbstractCallableTool.ts b/apps/backend/src/agent/tools/AbstractCallableTool.ts index 4e45440..b71cbf6 100644 --- a/apps/backend/src/agent/tools/AbstractCallableTool.ts +++ b/apps/backend/src/agent/tools/AbstractCallableTool.ts @@ -1,7 +1,7 @@ import type { DynamicStructuredTool } from '@langchain/core/tools'; import { PageContextProvider } from '@/indexer'; import type { CallableTool, CallableToolResult, CallableToolResultData, ToolResultElement } from '@/types'; -import { formantElementContextPath, type BaseElement } from '@flowforge/page-trail'; +import { formantElementContextPath, type TargetElement } from '@flowforge/page-trail'; export abstract class AbstractCallableTool implements CallableTool { readonly name: string; @@ -33,7 +33,7 @@ export abstract class AbstractCallableTool implements CallableTool { } } - protected getToolResultElement(element: BaseElement): ToolResultElement { + protected getToolResultElement(element: TargetElement): ToolResultElement { return { elementPath: formantElementContextPath(element.context.path), elementSectionName: element.context.sectionName ?? '', diff --git a/apps/backend/src/indexer/transformers/AbstractDocumentTransformer.ts b/apps/backend/src/indexer/transformers/AbstractDocumentTransformer.ts index 3535a2c..c7bdd48 100644 --- a/apps/backend/src/indexer/transformers/AbstractDocumentTransformer.ts +++ b/apps/backend/src/indexer/transformers/AbstractDocumentTransformer.ts @@ -1,6 +1,6 @@ import type { IndexableDocument, DocumentTransformer } from '@/types'; import { randomUUID } from 'crypto'; -import type { BaseElement, PageTrail } from '@flowforge/page-trail'; +import type { TargetElement, PageTrail } from '@flowforge/page-trail'; export abstract class AbstractDocumentTransformer implements DocumentTransformer { readonly name: string; @@ -15,7 +15,7 @@ export abstract class AbstractDocumentTransformer implements DocumentTransformer return randomUUID(); } - protected createDocument(content: string, el: BaseElement): IndexableDocument { + protected createDocument(content: string, el: TargetElement): IndexableDocument { return { id: this.createDocumentId(), content, diff --git a/apps/backend/src/types/documents.ts b/apps/backend/src/types/documents.ts index de90502..1b54d7f 100644 --- a/apps/backend/src/types/documents.ts +++ b/apps/backend/src/types/documents.ts @@ -1,10 +1,10 @@ -import type { BaseElement, ElementKind, PageTrail } from '@flowforge/page-trail'; +import type { ElementKind, TargetElement, PageTrail } from '@flowforge/page-trail'; export type DocumentType = ElementKind; export interface DocumentMetadata { type: DocumentType; - element: BaseElement; + element: TargetElement; } export interface Document { diff --git a/apps/extension/src/core/constants.ts b/apps/extension/src/core/constants.ts index 541279f..072ff2a 100644 --- a/apps/extension/src/core/constants.ts +++ b/apps/extension/src/core/constants.ts @@ -8,6 +8,7 @@ export const constants = { EXAMPLES_MAX_ITEMS: 5, EXAMPLES_MAX_QUESTION_LENGTH: 50, // extractors + CONTAINER_ELEMENTS_LIMIT: 50, CONTENT_ELEMENTS_LIMIT: 250, INTERACTIVE_ELEMENTS_LIMIT: 150, // highlighter diff --git a/apps/extension/src/page/components/Inspector/Inspector.tsx b/apps/extension/src/page/components/Inspector/Inspector.tsx index e0a7b41..e8ddc8f 100644 --- a/apps/extension/src/page/components/Inspector/Inspector.tsx +++ b/apps/extension/src/page/components/Inspector/Inspector.tsx @@ -1,6 +1,6 @@ import type { TargetedPointerEvent } from 'preact'; import { useEffect, useId, useState } from 'preact/hooks'; -import { BadgeInfo, BookOpenText, FileText, MousePointerClick } from 'lucide-preact'; +import { BadgeInfo, BookOpenText, ChartNoAxesColumn, FileText, ListTree, MousePointerClick } from 'lucide-preact'; import { getEventTarget } from '@/core/utils/dom'; import type { InspectorViewModel } from '@/page/hooks/usePage'; import { Button } from '@/shared/components/Button'; @@ -12,9 +12,11 @@ import { formatContentElement, formatInteractiveElement, generateSemanticMarkdow const inspectorTabs = [ { id: 'basics', label: 'Basics', icon: BadgeInfo }, + { id: 'container', label: 'Container', icon: ListTree }, { id: 'content', label: 'Content', icon: BookOpenText }, { id: 'interactive', label: 'Interactive', icon: MousePointerClick }, { id: 'semanticView', label: 'Semantic view', icon: FileText }, + { id: 'metadata', label: 'Metadata', icon: ChartNoAxesColumn }, ] as const; type InspectorTabId = (typeof inspectorTabs)[number]['id']; @@ -105,6 +107,7 @@ export function Inspector({ pageTrail, close }: InspectorViewModel) { aria-labelledby={getTabId(activeTab)} > {activeTab === 'basics' && } + {activeTab === 'container' && } {activeTab === 'content' && ( )} {activeTab === 'semanticView' && } + {activeTab === 'metadata' && }
diff --git a/apps/extension/src/page/components/Inspector/components/Metadata/PageMetadata.tsx b/apps/extension/src/page/components/Inspector/components/Metadata/PageMetadata.tsx index 85d7739..f699a51 100644 --- a/apps/extension/src/page/components/Inspector/components/Metadata/PageMetadata.tsx +++ b/apps/extension/src/page/components/Inspector/components/Metadata/PageMetadata.tsx @@ -35,7 +35,7 @@ export function PageMetadata({ metadata }: PageMetadataProps) { ) : ( <>{metadata.interactiveElements} )}{' '} - interactive elements · {metadata.durationMs}ms + interactive elements · {metadata.performance.totalMs}ms
); } diff --git a/apps/extension/src/page/hooks/usePage.ts b/apps/extension/src/page/hooks/usePage.ts index 314bb8e..6211b1c 100644 --- a/apps/extension/src/page/hooks/usePage.ts +++ b/apps/extension/src/page/hooks/usePage.ts @@ -15,6 +15,7 @@ import { PageTrailCollector } from '@flowforge/page-trail'; function collectPageTrail(): PageTrail { return PageTrailCollector.collectFor(window, document, { + containerElementsLimit: constants.CONTAINER_ELEMENTS_LIMIT, contentElementsLimit: constants.CONTENT_ELEMENTS_LIMIT, interactiveElementsLimit: constants.INTERACTIVE_ELEMENTS_LIMIT, getElementDataId: getOrCreateDataId, diff --git a/packages/page-trail/src/collector/ElementRegistry.ts b/packages/page-trail/src/collector/ElementRegistry.ts new file mode 100644 index 0000000..5a77608 --- /dev/null +++ b/packages/page-trail/src/collector/ElementRegistry.ts @@ -0,0 +1,21 @@ +import type { ElementDataId } from '../types/index.ts'; + +export class ElementRegistry { + private readonly elementByDataId = new Map(); + + constructor(private readonly produceDataId: (el: Element) => ElementDataId) {} + + register(el: Element): ElementDataId { + const dataId = this.produceDataId(el); + this.elementByDataId.set(dataId, el); + return dataId; + } + + get(dataId: ElementDataId): Element | undefined { + return this.elementByDataId.get(dataId); + } + + clear(): void { + this.elementByDataId.clear(); + } +} diff --git a/packages/page-trail/src/collector/PageTrailCollector.test.ts b/packages/page-trail/src/collector/PageTrailCollector.test.ts index 7be8e12..8192cd5 100644 --- a/packages/page-trail/src/collector/PageTrailCollector.test.ts +++ b/packages/page-trail/src/collector/PageTrailCollector.test.ts @@ -31,7 +31,7 @@ describe('PageTrailCollector', () => { }, }); expect(model.metadata.collectedAt).toBeTypeOf('number'); - expect(model.metadata.durationMs).toBeTypeOf('number'); + expect(model.metadata.performance.totalMs).toBeTypeOf('number'); }); it('normalizes page basics text', () => { @@ -72,7 +72,7 @@ describe('PageTrailCollector', () => { tag: 'h1', text: 'Welcome', dataId: 'title', - cssSelector: '#title', + cssSelector: undefined, }), expect.objectContaining({ kind: 'content', @@ -80,7 +80,7 @@ describe('PageTrailCollector', () => { tag: 'p', text: 'Useful paragraph text', dataId: 'intro', - cssSelector: '#intro', + cssSelector: undefined, }), ]), ); @@ -108,7 +108,7 @@ describe('PageTrailCollector', () => { type: 'button', role: 'button', dataId: 'save', - cssSelector: '#save', + cssSelector: undefined, labels: [{ source: 'aria-label', value: 'Save changes' }], }), expect.objectContaining({ @@ -116,7 +116,7 @@ describe('PageTrailCollector', () => { type: 'link', role: 'link', dataId: 'docs', - cssSelector: '#docs', + cssSelector: undefined, link: { type: 'internal', href: 'http://localhost:3000/docs', @@ -127,7 +127,7 @@ describe('PageTrailCollector', () => { type: 'input', role: 'textbox', dataId: 'email', - cssSelector: '#email', + cssSelector: undefined, labels: [{ source: 'placeholder', value: 'Email' }], }), ]), @@ -174,6 +174,24 @@ describe('PageTrailCollector', () => { expect(model.interactive[0]).toEqual(expect.objectContaining({ dataId: 'button' })); }); + it('reports container metadata without scoring limit totals', () => { + document.body.innerHTML = ` +
+
+
+ `; + markVisible('#main'); + markVisible('#section'); + + const model = collect(); + + expect(model.metadata.containerElements).toBe(2); + expect(model.metadata.containerMaxDepth).toBe(2); + expect(model.metadata).not.toHaveProperty('containerElementsTotal'); + expect(model.metadata).not.toHaveProperty('containerElementsLimitReached'); + expect(model.container[0]).not.toHaveProperty('importanceScore'); + }); + it('keeps cssSelector undefined when css selector resolver is not configured', () => { document.body.innerHTML = ``; markVisible('#save'); @@ -201,7 +219,6 @@ describe('PageTrailCollector', () => { function collect(options: Partial[2]> = {}) { return new PageTrailCollector(window, document, { getElementDataId: (el) => el.id, - getElementCssSelector: (el) => `#${el.id}`, ...options, }).collect(); } diff --git a/packages/page-trail/src/collector/PageTrailCollector.ts b/packages/page-trail/src/collector/PageTrailCollector.ts index 9a37d47..d36371a 100644 --- a/packages/page-trail/src/collector/PageTrailCollector.ts +++ b/packages/page-trail/src/collector/PageTrailCollector.ts @@ -1,83 +1,76 @@ -import type { ContentElement, ElementIdentifier, InteractiveElement, PageBasics, PageTrail } from '../types/index.ts'; +import type { ContentElement, InteractiveElement, PageBasics, PageTrail } from '../types/index.ts'; -import { getElementLabels } from './extractors/primitive/label.ts'; -import { getInteractiveRole, roleToInteractiveElementType } from './extractors/primitive/role.ts'; -import { getElementBoundingBox, isAboveTheFold, isElementVisible, isInViewport } from './extractors/primitive/view.ts'; -import { getInteractiveElementState } from './extractors/primitive/state.ts'; -import { getElementText } from './extractors/primitive/text.ts'; -import { getElementLink } from './extractors/primitive/link.ts'; -import { getElementContext } from './extractors/context.ts'; -import { isSensitiveElement } from './extractors/primitive/sensitive.ts'; -import { scoreContentElement, scoreInteractiveElement } from './importance/scoring.ts'; -import { type TopElements, topElements } from './importance/topEl.ts'; -import type { InteractiveElementScoringData } from './importance/interactive.ts'; -import type { ContentElementScoringData } from './importance/content.ts'; -import { normalizeText } from '../utils/index.ts'; - -// constants -const CONTENT_MIN_TEXT_LENGTH = 5; +import { type TopElements } from './importance/topEl.ts'; +import { ContainerTree } from './extractors/ContainerTree.ts'; +import { ElementRegistry } from './ElementRegistry.ts'; +import { extractContentElements } from './extractors/content.ts'; +import { extractPageBasics } from './extractors/basics.ts'; +import { extractInteractiveElements } from './extractors/interactive.ts'; export interface CollectorOptions { + /** Maximum number of content elements to keep after importance scoring. */ contentElementsLimit?: number; + /** Maximum number of interactive elements to keep after importance scoring. */ interactiveElementsLimit?: number; + /** Returns the stable identifier used to link extracted records back to DOM elements. */ getElementDataId: (el: Element) => string; - // For example: css-selector-generator - getElementCssSelector?: (el: Element) => string; } type ResolvedCollectorOptions = Required> & - Pick; + Pick; /** - * Collects a normalized PageTrail from the DOM + * Orchestrates PageTrail extraction for a document. * * TODO: implement a cache, but with dataId ref consistency * TODO: provide a plugins API to extend the collector * - * Extracts page metadata, content, and interactive elements, and assigns - * stable `dataId` identifiers to elements for downstream usage. + * The collector owns shared extraction state, delegates DOM scanning to + * specialized extractors, and combines their results into a normalized + * `PageTrail` with collection metadata. */ export class PageTrailCollector { - // window.location.href - reads the current page URL - // window.innerWidth - reads viewport width - // window.innerHeight - reads viewport height - // window.scrollY - reads current vertical scroll position - // window.getComputedStyle(element) - checks computed CSS styles for visibility private readonly window: Window; - // document.title - reads the page title - // document.querySelector('meta[name="description"]') - finds the meta description element - // document.documentElement.lang - reads the page language from - // document.documentElement.scrollHeight - reads the full scrollable page height - // document.querySelectorAll(selector) - finds content and interactive elements by CSS selector - // document.getElementById(id) - resolves IDs from aria-labelledby to label elements - // document.querySelector('label[for="..."]') - finds a