From 99ad7b1f7ea3066519f3a5ab5f0e9952a175a626 Mon Sep 17 00:00:00 2001 From: Shanto Islam Date: Wed, 2 Sep 2026 09:29:49 +0600 Subject: [PATCH 01/13] refactor: rebuild agentic webview as modular SDK --- .github/workflows/build-demo.yml | 18 +- .github/workflows/ci.yml | 69 +- .github/workflows/publish.yml | 45 +- .gitignore | 2 - AGENTS.md | 174 +- CONTRIBUTING.md | 151 +- README.md | 146 +- agent-tools/build.gradle.kts | 29 + .../agenticwebview/tools/AgentTool.kt | 32 + .../tools/StandardBrowserTools.kt | 517 ++++++ .../tools/StandardBrowserToolsTest.kt | 208 +++ agentic-webview/.gitignore | 2 - agentic-webview/build.gradle.kts | 124 -- agentic-webview/consumer-rules.pro | 7 - agentic-webview/proguard-rules.pro | 1 - .../androidTest/assets/basic_interactive.html | 20 - .../agenticwebview/ConcurrencyStressTest.kt | 49 - .../agenticwebview/ControllerCoreTest.kt | 88 - .../agenticwebview/CoordinateMappingTest.kt | 57 - .../agenticwebview/DomEdgeCaseTest.kt | 88 - .../agenticwebview/EnhancedFeaturesTest.kt | 300 ---- .../agenticwebview/ErrorRecoveryTest.kt | 56 - .../agenticwebview/InputCompatibilityTest.kt | 50 - .../agenticwebview/MockWebServerRule.kt | 49 - .../agenticwebview/SecurityTest.kt | 43 - .../agenticwebview/StabilityTest.kt | 77 - agentic-webview/src/debug/AndroidManifest.xml | 9 - .../agenticwebview/TestActivity.kt | 14 - .../agenticwebview/AgenticWebController.kt | 514 ------ .../agenticwebview/AgenticWebView.kt | 329 ---- .../agenticwebview/AgenticWebViewCompose.kt | 47 - .../config/AgenticWebViewConfig.kt | 71 - .../agenticwebview/internal/JsEvaluator.kt | 87 - .../agenticwebview/internal/JsUtils.kt | 16 - .../internal/ScreenshotCapture.kt | 87 - .../agenticwebview/internal/SdkLogger.kt | 21 - .../agenticwebview/models/AgentAction.kt | 27 - .../agenticwebview/models/AgentResult.kt | 20 - .../agenticwebview/models/AgentState.kt | 27 - .../agenticwebview/models/DropdownOption.kt | 10 - .../models/PageLifecycleState.kt | 13 - .../src/main/res/drawable/ic_agentic_logo.xml | 44 - .../agenticwebview/internal/JsUtilsTest.kt | 22 - .../app/ExampleInstrumentedTest.kt | 24 - .../agenticwebview/app/MainActivity.kt | 508 ------ .../app/agent/AgenticWebviewTools.kt | 91 - .../app/model/AgentSettingsDao.kt | 16 - .../app/model/AgentSettingsEntity.kt | 13 - .../agenticwebview/app/model/AppDatabase.kt | 28 - .../app/ui/AgenticWebViewModel.kt | 181 -- .../agenticwebview/app/ui/theme/Color.kt | 219 --- .../agenticwebview/app/ui/theme/Theme.kt | 280 --- .../agenticwebview/app/ui/theme/Type.kt | 34 - app/src/main/res/values/colors.xml | 10 - app/src/main/res/values/strings.xml | 3 - app/src/main/res/xml/backup_rules.xml | 13 - .../main/res/xml/data_extraction_rules.xml | 19 - .../agenticwebview/app/ExampleUnitTest.kt | 17 - browser-api/build.gradle.kts | 29 + .../agenticwebview/api/BrowserCommand.kt | 141 ++ .../api/BrowserConfiguration.kt | 182 ++ .../agenticwebview/api/BrowserObservation.kt | 178 ++ .../agenticwebview/api/BrowserResult.kt | 116 ++ .../agenticwebview/api/BrowserSession.kt | 94 + .../agenticwebview/api/Identifiers.kt | 45 + .../api/BrowserConfigurationTest.kt | 59 + .../agenticwebview/api/IdentifiersTest.kt | 24 + browser-compose/build.gradle.kts | 49 + browser-compose/src/main/AndroidManifest.xml | 1 + .../compose/AgenticBrowserCompose.kt | 68 + browser-webview/build.gradle.kts | 62 + browser-webview/consumer-rules.pro | 6 + .../src/androidTest/AndroidManifest.xml | 4 + .../AgenticBrowserHostInstrumentedTest.kt | 59 + browser-webview/src/main/AndroidManifest.xml | 3 + .../webview/AgenticBrowserHost.kt | 81 + .../webview/AndroidAgenticBrowserSession.kt | 1325 ++++++++++++++ .../webview/BrowserHostDelegate.kt | 50 + .../webview/BrowserLifecycleReducer.kt | 87 + .../webview/NavigationPolicyEvaluator.kt | 38 + .../webview/PixelCopyScreenshotProvider.kt | 268 +++ .../webview/protocol/RuntimeProtocol.kt | 88 + .../webview/protocol/RuntimeProtocolBridge.kt | 95 + .../webview/protocol/RuntimeProtocolCodec.kt | 58 + .../protocol/RuntimeProtocolGateway.kt | 94 + .../protocol/RuntimeRequestRegistry.kt | 115 ++ .../protocol/WebViewRuntimeTransport.kt | 72 + .../webview/BrowserLifecycleReducerTest.kt | 61 + .../webview/NavigationPolicyEvaluatorTest.kt | 61 + .../protocol/RuntimeProtocolCodecTest.kt | 94 + .../protocol/RuntimeProtocolGatewayTest.kt | 77 + .../protocol/RuntimeRequestRegistryTest.kt | 103 ++ build.gradle.kts | 38 +- docs/adr/0001-no-compatibility-rewrite.md | 16 + docs/agent-perception.md | 90 - docs/agent-tools.md | 29 + docs/architecture-refactoring-plan.md | 1595 +++++++++++++++++ docs/architecture.md | 22 + docs/best-practices.md | 82 - docs/commands.md | 14 + docs/compose.md | 18 + docs/diagnostics.md | 15 + docs/frames-shadow-dom.md | 7 + docs/getting-started.md | 100 +- docs/koog.md | 12 + docs/lifecycle.md | 7 + docs/observations.md | 10 + docs/privacy-prompt-injection.md | 16 + docs/release.md | 10 + docs/security.md | 20 + docs/testing.md | 18 + docs/views.md | 32 + gradle.properties | 2 +- gradle/libs.versions.toml | 19 +- gradle/publishing.gradle.kts | 39 + integrations/jsonrpc/build.gradle.kts | 27 + .../jsonrpc/JsonRpcBrowserToolServer.kt | 106 ++ .../jsonrpc/JsonRpcBrowserToolServerTest.kt | 86 + integrations/koog/build.gradle.kts | 27 + .../integrations/koog/KoogBrowserTools.kt | 136 ++ .../integrations/koog/KoogBrowserToolsTest.kt | 50 + protocol-fixtures/v1/request-ping.json | 9 + .../v1/response-ping-success.json | 11 + .../v1/response-runtime-error.json | 15 + {app => samples/android}/.gitignore | 0 {app => samples/android}/build.gradle.kts | 32 +- {app => samples/android}/proguard-rules.pro | 0 .../android}/src/main/AndroidManifest.xml | 12 +- .../agenticwebview/app/MainActivity.kt | 113 ++ .../agenticwebview/app/ui/theme/Theme.kt | 15 + .../res/drawable/ic_launcher_background.xml | 0 .../res/drawable/ic_launcher_foreground.xml | 0 .../main/res/mipmap-anydpi/ic_launcher.xml | 0 .../res/mipmap-anydpi/ic_launcher_round.xml | 0 .../android/src/main/res/values/strings.xml | 3 + .../android}/src/main/res/values/themes.xml | 0 scripts/check-artifact-sizes.mjs | 49 + scripts/validate-architecture.mjs | 128 ++ scripts/validate-doc-links.mjs | 29 + settings.gradle.kts | 9 +- test-pages/README.md | 5 + test-pages/fixtures/complex.html | 50 + test-pages/fixtures/frame.html | 10 + test-pages/fixtures/nested-frame.html | 5 + test-pages/server/server.mjs | 29 + web-injector/LICENSE | 201 --- web-injector/package.json | 19 - web-injector/src/bridge.ts | 34 - web-injector/src/buildDomTree.ts | 667 ------- web-injector/src/cssSelector.ts | 93 - web-injector/src/domParser.test.ts | 215 --- web-injector/src/domParser.ts | 244 --- web-injector/src/elementHash.ts | 66 - web-injector/src/iframeBus.ts | 84 - web-injector/src/index.ts | 313 ---- web-injector/src/interaction.ts | 259 --- web-injector/src/serializer.ts | 107 -- web-injector/src/stability.ts | 44 - {web-injector => web-runtime}/.gitignore | 0 {agentic-webview => web-runtime}/LICENSE | 0 {web-injector => web-runtime}/jest.config.js | 0 .../package-lock.json | 8 +- web-runtime/package.json | 21 + web-runtime/src/index.ts | 85 + web-runtime/src/protocol/dispatcher.test.ts | 119 ++ web-runtime/src/protocol/dispatcher.ts | 173 ++ web-runtime/src/protocol/types.ts | 44 + web-runtime/src/protocol/validation.ts | 140 ++ .../src/runtime/actionExecutor.test.ts | 232 +++ web-runtime/src/runtime/actionExecutor.ts | 636 +++++++ .../src/runtime/elementRegistry.test.ts | 42 + web-runtime/src/runtime/elementRegistry.ts | 52 + .../src/runtime/experimentalPagePatches.ts | 40 + web-runtime/src/runtime/frameRegistry.test.ts | 71 + web-runtime/src/runtime/frameRegistry.ts | 227 +++ .../src/runtime/revisionTracker.test.ts | 29 + web-runtime/src/runtime/revisionTracker.ts | 56 + .../src/runtime/semanticObserver.test.ts | 178 ++ web-runtime/src/runtime/semanticObserver.ts | 607 +++++++ web-runtime/src/runtime/semanticRuntime.ts | 88 + {web-injector => web-runtime}/tsconfig.json | 0 website/package-lock.json | 1542 ++++------------ website/package.json | 16 +- website/src/App.tsx | 11 +- website/src/components/DocPage.tsx | 35 - website/src/components/InlineCode.tsx | 17 - website/src/hooks/useLatestRelease.ts | 35 - website/src/pages/CanonicalDocs.tsx | 62 + website/src/pages/DocumentationLayout.tsx | 148 -- website/src/pages/Home.tsx | 173 +- website/src/pages/docs/AgentIntegration.tsx | 112 -- website/src/pages/docs/BestPractices.tsx | 112 -- website/src/pages/docs/IntegrationGuide.tsx | 107 -- 193 files changed, 11090 insertions(+), 8744 deletions(-) create mode 100644 agent-tools/build.gradle.kts create mode 100644 agent-tools/src/main/kotlin/dev/shantoislam/agenticwebview/tools/AgentTool.kt create mode 100644 agent-tools/src/main/kotlin/dev/shantoislam/agenticwebview/tools/StandardBrowserTools.kt create mode 100644 agent-tools/src/test/kotlin/dev/shantoislam/agenticwebview/tools/StandardBrowserToolsTest.kt delete mode 100644 agentic-webview/.gitignore delete mode 100644 agentic-webview/build.gradle.kts delete mode 100644 agentic-webview/consumer-rules.pro delete mode 100644 agentic-webview/proguard-rules.pro delete mode 100644 agentic-webview/src/androidTest/assets/basic_interactive.html delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ConcurrencyStressTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ControllerCoreTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/CoordinateMappingTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/DomEdgeCaseTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/EnhancedFeaturesTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ErrorRecoveryTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/InputCompatibilityTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/MockWebServerRule.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/SecurityTest.kt delete mode 100644 agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/StabilityTest.kt delete mode 100644 agentic-webview/src/debug/AndroidManifest.xml delete mode 100644 agentic-webview/src/debug/java/dev/shantoislam/agenticwebview/TestActivity.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebController.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebView.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebViewCompose.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/config/AgenticWebViewConfig.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsEvaluator.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsUtils.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/ScreenshotCapture.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/SdkLogger.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentAction.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentResult.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentState.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/DropdownOption.kt delete mode 100644 agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/PageLifecycleState.kt delete mode 100644 agentic-webview/src/main/res/drawable/ic_agentic_logo.xml delete mode 100644 agentic-webview/src/test/java/dev/shantoislam/agenticwebview/internal/JsUtilsTest.kt delete mode 100644 app/src/androidTest/java/dev/shantoislam/agenticwebview/app/ExampleInstrumentedTest.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/MainActivity.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/agent/AgenticWebviewTools.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsDao.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsEntity.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/model/AppDatabase.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/ui/AgenticWebViewModel.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Color.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Theme.kt delete mode 100644 app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Type.kt delete mode 100644 app/src/main/res/values/colors.xml delete mode 100644 app/src/main/res/values/strings.xml delete mode 100644 app/src/main/res/xml/backup_rules.xml delete mode 100644 app/src/main/res/xml/data_extraction_rules.xml delete mode 100644 app/src/test/java/dev/shantoislam/agenticwebview/app/ExampleUnitTest.kt create mode 100644 browser-api/build.gradle.kts create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserCommand.kt create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfiguration.kt create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserObservation.kt create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserResult.kt create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserSession.kt create mode 100644 browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/Identifiers.kt create mode 100644 browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfigurationTest.kt create mode 100644 browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/IdentifiersTest.kt create mode 100644 browser-compose/build.gradle.kts create mode 100644 browser-compose/src/main/AndroidManifest.xml create mode 100644 browser-compose/src/main/kotlin/dev/shantoislam/agenticwebview/compose/AgenticBrowserCompose.kt create mode 100644 browser-webview/build.gradle.kts create mode 100644 browser-webview/consumer-rules.pro create mode 100644 browser-webview/src/androidTest/AndroidManifest.xml create mode 100644 browser-webview/src/androidTest/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHostInstrumentedTest.kt create mode 100644 browser-webview/src/main/AndroidManifest.xml create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHost.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AndroidAgenticBrowserSession.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserHostDelegate.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducer.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluator.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/PixelCopyScreenshotProvider.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocol.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolBridge.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodec.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGateway.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistry.kt create mode 100644 browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/WebViewRuntimeTransport.kt create mode 100644 browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducerTest.kt create mode 100644 browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluatorTest.kt create mode 100644 browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodecTest.kt create mode 100644 browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGatewayTest.kt create mode 100644 browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistryTest.kt create mode 100644 docs/adr/0001-no-compatibility-rewrite.md delete mode 100644 docs/agent-perception.md create mode 100644 docs/agent-tools.md create mode 100644 docs/architecture-refactoring-plan.md create mode 100644 docs/architecture.md delete mode 100644 docs/best-practices.md create mode 100644 docs/commands.md create mode 100644 docs/compose.md create mode 100644 docs/diagnostics.md create mode 100644 docs/frames-shadow-dom.md create mode 100644 docs/koog.md create mode 100644 docs/lifecycle.md create mode 100644 docs/observations.md create mode 100644 docs/privacy-prompt-injection.md create mode 100644 docs/release.md create mode 100644 docs/security.md create mode 100644 docs/testing.md create mode 100644 docs/views.md create mode 100644 gradle/publishing.gradle.kts create mode 100644 integrations/jsonrpc/build.gradle.kts create mode 100644 integrations/jsonrpc/src/main/kotlin/dev/shantoislam/agenticwebview/integrations/jsonrpc/JsonRpcBrowserToolServer.kt create mode 100644 integrations/jsonrpc/src/test/kotlin/dev/shantoislam/agenticwebview/integrations/jsonrpc/JsonRpcBrowserToolServerTest.kt create mode 100644 integrations/koog/build.gradle.kts create mode 100644 integrations/koog/src/main/kotlin/dev/shantoislam/agenticwebview/integrations/koog/KoogBrowserTools.kt create mode 100644 integrations/koog/src/test/kotlin/dev/shantoislam/agenticwebview/integrations/koog/KoogBrowserToolsTest.kt create mode 100644 protocol-fixtures/v1/request-ping.json create mode 100644 protocol-fixtures/v1/response-ping-success.json create mode 100644 protocol-fixtures/v1/response-runtime-error.json rename {app => samples/android}/.gitignore (100%) rename {app => samples/android}/build.gradle.kts (62%) rename {app => samples/android}/proguard-rules.pro (100%) rename {app => samples/android}/src/main/AndroidManifest.xml (73%) create mode 100644 samples/android/src/main/java/dev/shantoislam/agenticwebview/app/MainActivity.kt create mode 100644 samples/android/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Theme.kt rename {app => samples/android}/src/main/res/drawable/ic_launcher_background.xml (100%) rename {app => samples/android}/src/main/res/drawable/ic_launcher_foreground.xml (100%) rename {app => samples/android}/src/main/res/mipmap-anydpi/ic_launcher.xml (100%) rename {app => samples/android}/src/main/res/mipmap-anydpi/ic_launcher_round.xml (100%) create mode 100644 samples/android/src/main/res/values/strings.xml rename {app => samples/android}/src/main/res/values/themes.xml (100%) create mode 100644 scripts/check-artifact-sizes.mjs create mode 100644 scripts/validate-architecture.mjs create mode 100644 scripts/validate-doc-links.mjs create mode 100644 test-pages/README.md create mode 100644 test-pages/fixtures/complex.html create mode 100644 test-pages/fixtures/frame.html create mode 100644 test-pages/fixtures/nested-frame.html create mode 100644 test-pages/server/server.mjs delete mode 100644 web-injector/LICENSE delete mode 100644 web-injector/package.json delete mode 100644 web-injector/src/bridge.ts delete mode 100644 web-injector/src/buildDomTree.ts delete mode 100644 web-injector/src/cssSelector.ts delete mode 100644 web-injector/src/domParser.test.ts delete mode 100644 web-injector/src/domParser.ts delete mode 100644 web-injector/src/elementHash.ts delete mode 100644 web-injector/src/iframeBus.ts delete mode 100644 web-injector/src/index.ts delete mode 100644 web-injector/src/interaction.ts delete mode 100644 web-injector/src/serializer.ts delete mode 100644 web-injector/src/stability.ts rename {web-injector => web-runtime}/.gitignore (100%) rename {agentic-webview => web-runtime}/LICENSE (100%) rename {web-injector => web-runtime}/jest.config.js (100%) rename {web-injector => web-runtime}/package-lock.json (99%) create mode 100644 web-runtime/package.json create mode 100644 web-runtime/src/index.ts create mode 100644 web-runtime/src/protocol/dispatcher.test.ts create mode 100644 web-runtime/src/protocol/dispatcher.ts create mode 100644 web-runtime/src/protocol/types.ts create mode 100644 web-runtime/src/protocol/validation.ts create mode 100644 web-runtime/src/runtime/actionExecutor.test.ts create mode 100644 web-runtime/src/runtime/actionExecutor.ts create mode 100644 web-runtime/src/runtime/elementRegistry.test.ts create mode 100644 web-runtime/src/runtime/elementRegistry.ts create mode 100644 web-runtime/src/runtime/experimentalPagePatches.ts create mode 100644 web-runtime/src/runtime/frameRegistry.test.ts create mode 100644 web-runtime/src/runtime/frameRegistry.ts create mode 100644 web-runtime/src/runtime/revisionTracker.test.ts create mode 100644 web-runtime/src/runtime/revisionTracker.ts create mode 100644 web-runtime/src/runtime/semanticObserver.test.ts create mode 100644 web-runtime/src/runtime/semanticObserver.ts create mode 100644 web-runtime/src/runtime/semanticRuntime.ts rename {web-injector => web-runtime}/tsconfig.json (100%) delete mode 100644 website/src/components/DocPage.tsx delete mode 100644 website/src/components/InlineCode.tsx delete mode 100644 website/src/hooks/useLatestRelease.ts create mode 100644 website/src/pages/CanonicalDocs.tsx delete mode 100644 website/src/pages/DocumentationLayout.tsx delete mode 100644 website/src/pages/docs/AgentIntegration.tsx delete mode 100644 website/src/pages/docs/BestPractices.tsx delete mode 100644 website/src/pages/docs/IntegrationGuide.tsx diff --git a/.github/workflows/build-demo.yml b/.github/workflows/build-demo.yml index 5a49397..e7f4f85 100644 --- a/.github/workflows/build-demo.yml +++ b/.github/workflows/build-demo.yml @@ -26,21 +26,19 @@ jobs: with: node-version: '20' cache: 'npm' - cache-dependency-path: web-injector/package-lock.json + cache-dependency-path: web-runtime/package-lock.json - - name: Build web-injector - working-directory: web-injector + - name: Check web runtime + working-directory: web-runtime run: | npm ci - npm run build - - - name: Copy web-injector bundle to assets - run: | - mkdir -p agentic-webview/src/main/assets - cp web-injector/dist/agentic_core.min.js agentic-webview/src/main/assets/ + npm run check - name: Make Gradle wrapper executable run: chmod +x ./gradlew + - name: Validate architecture boundaries + run: node scripts/validate-architecture.mjs + - name: Build demo app - run: ./gradlew :app:assembleDebug + run: ./gradlew :samples:android:assembleDebug diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1baf3fd..f103f7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,24 +26,75 @@ jobs: with: node-version: '20' cache: 'npm' - cache-dependency-path: web-injector/package-lock.json + cache-dependency-path: web-runtime/package-lock.json - - name: Build web-injector - working-directory: web-injector + - name: Check and build web runtime + working-directory: web-runtime run: | npm ci - npm run build + npm run check - - name: Copy web-injector bundle to assets + - name: Build documentation website + working-directory: website run: | - mkdir -p agentic-webview/src/main/assets - cp web-injector/dist/agentic_core.min.js agentic-webview/src/main/assets/ + npm ci + npm run lint + npm run build + + - name: Validate canonical documentation links + run: node scripts/validate-doc-links.mjs + + - name: Validate architecture boundaries + run: node scripts/validate-architecture.mjs - name: Make Gradle wrapper executable run: chmod +x ./gradlew - name: Build library - run: ./gradlew :agentic-webview:assembleRelease + run: ./gradlew :browser-api:test :browser-api:jar :agent-tools:test :agent-tools:jar :integrations:jsonrpc:test :integrations:jsonrpc:jar :integrations:koog:check :integrations:koog:jar :browser-webview:testDebugUnitTest :browser-webview:assembleRelease :browser-compose:testDebugUnitTest :browser-compose:assembleRelease :samples:android:assembleDebug + + - name: Enforce artifact-size budgets + run: node scripts/check-artifact-sizes.mjs --require-built - name: Lint - run: ./gradlew :agentic-webview:lint + run: ./gradlew :browser-webview:lint :browser-compose:lint :samples:android:lint + + - name: Validate publication metadata and consumer rules + run: ./gradlew generatePomFileForMavenPublication :browser-webview:assembleRelease + + instrumented: + name: Android instrumentation + runs-on: ubuntu-latest + needs: build + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install runtime dependencies + working-directory: web-runtime + run: npm ci + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Run deterministic WebView instrumentation + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 35 + arch: x86_64 + script: ./gradlew :browser-webview:connectedDebugAndroidTest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 951e000..0edc655 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -28,18 +28,16 @@ jobs: with: node-version: '20' cache: 'npm' - cache-dependency-path: web-injector/package-lock.json + cache-dependency-path: web-runtime/package-lock.json - - name: Build web-injector - working-directory: web-injector + - name: Check and build web runtime + working-directory: web-runtime run: | npm ci - npm run build + npm run check - - name: Copy web-injector bundle to assets - run: | - mkdir -p agentic-webview/src/main/assets - cp web-injector/dist/agentic_core.min.js agentic-webview/src/main/assets/ + - name: Validate architecture boundaries + run: node scripts/validate-architecture.mjs - name: Extract version from tag id: version @@ -48,8 +46,24 @@ jobs: - name: Make Gradle wrapper executable run: chmod +x ./gradlew - - name: Publish to Maven Central - run: ./gradlew :agentic-webview:publishAndReleaseToMavenCentral --no-configuration-cache + - name: Validate release artifacts + run: ./gradlew :browser-api:test :browser-api:jar :agent-tools:test :agent-tools:jar :integrations:jsonrpc:test :integrations:jsonrpc:jar :integrations:koog:check :integrations:koog:jar :browser-webview:testDebugUnitTest :browser-webview:assembleRelease :browser-compose:assembleRelease :samples:android:assembleDebug + env: + ORG_GRADLE_PROJECT_VERSION_NAME: ${{ steps.version.outputs.VERSION }} + + - name: Enforce artifact-size budgets + run: node scripts/check-artifact-sizes.mjs --require-built + + - name: Publish all SDK modules to Maven Central + run: >- + ./gradlew + :browser-api:publishAndReleaseToMavenCentral + :browser-webview:publishAndReleaseToMavenCentral + :browser-compose:publishAndReleaseToMavenCentral + :agent-tools:publishAndReleaseToMavenCentral + :integrations:koog:publishAndReleaseToMavenCentral + :integrations:jsonrpc:publishAndReleaseToMavenCentral + --no-configuration-cache env: ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_CENTRAL_USERNAME }} ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} @@ -58,16 +72,21 @@ jobs: ORG_GRADLE_PROJECT_VERSION_NAME: ${{ steps.version.outputs.VERSION }} - name: Build demo app APK (debug signed) - run: ./gradlew :app:assembleDebug + run: ./gradlew :samples:android:assembleDebug env: ORG_GRADLE_PROJECT_VERSION_NAME: ${{ steps.version.outputs.VERSION }} - name: Rename APK with version - run: mv app/build/outputs/apk/debug/app-debug.apk app/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk + run: mv samples/android/build/outputs/apk/debug/android-debug.apk samples/android/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk + + - name: Create demo checksum + run: sha256sum samples/android/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk > samples/android/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk.sha256 - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: generate_release_notes: true tag_name: ${{ github.ref_name }} - files: app/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk + files: | + samples/android/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk + samples/android/build/outputs/apk/debug/agentic-webview-demo-${{ steps.version.outputs.VERSION }}.apk.sha256 diff --git a/.gitignore b/.gitignore index ea9b926..edb6622 100644 --- a/.gitignore +++ b/.gitignore @@ -67,7 +67,5 @@ fabric.properties .idea/tasks.xml .idea/dictionaries -# Generated assets -agentic-webview/src/main/assets/agentic_core.min.js node_modules/ dist/ diff --git a/AGENTS.md b/AGENTS.md index f3f0d8a..e13d974 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,156 +1,34 @@ -# AGENTS.md - Instructions for AI Coding Agents +# Repository instructions -## Project Context -- **Description**: Agentic WebView SDK - An Android library providing an accessibility-tree-based perception layer and native interaction pipeline for LLM-powered web agents. -- **Primary Stack**: Kotlin (Android SDK), TypeScript (DOM Injection), Gradle, npm, esbuild. -- **Package Manager**: Use `./gradlew` for Android/Kotlin and `npm` for the `web-injector` module. +## Tooling -## Essential Commands -- **Build Full Project**: `./gradlew assembleDebug` -- **Build Web Injector**: `cd web-injector && npm run build` (Generates `dist/agentic_core.min.js`) -- **Run Instrumented Tests**: `./gradlew :agentic-webview:connectedAndroidTest` -- **Lint Android**: `./gradlew lint` -- **Run TS Tests**: `cd web-injector && npm test` +- Android/Kotlin: `./gradlew` +- Page runtime: `npm` in `web-runtime/` +- Runtime check: `npm run check` +- Architecture check: `node scripts/validate-architecture.mjs` +- Documentation check: `node scripts/validate-doc-links.mjs` +- Full deterministic check: `./gradlew check :browser-api:jar :agent-tools:jar :integrations:jsonrpc:jar :integrations:koog:jar :browser-webview:assembleRelease :browser-compose:assembleRelease :samples:android:assembleDebug` +- Built artifact budgets: `node scripts/check-artifact-sizes.mjs --require-built` -## Project Structure -- `agentic-webview/`: Core Android library. - - `src/main/java/`: Kotlin implementation of the custom WebView, Controller, and models. - - `src/main/assets/`: Contains the bundled `agentic_core.min.js` (do not edit directly). - - `src/androidTest/`: Comprehensive instrumented test suite using MockWebServer. - - `src/test/`: Unit tests. - - `config/`: `AgenticWebViewConfig` data class with `Builder` pattern. - - `internal/`: Internal helpers — `JsEvaluator.kt`, `JsUtils.kt`, `ScreenshotCapture.kt`, `SdkLogger.kt`. - - `models/`: `AgentAction.kt`, `AgentResult.kt`, `AgentState.kt`, `DropdownOption.kt`, `PageLifecycleState.kt`. -- `web-injector/`: TypeScript project for in-page DOM parsing and interaction. - - `src/index.ts`: Main entry point — creates `AgenticEngine`, wires up all modules, sets up anti-detection, mutation observers, and iframe bus. Exports `window.__AgenticInternal`. - - `src/buildDomTree.ts`: Core DOM tree engine. WeakMap caching, cursor-based interactive detection, isTopElement, XPath generation, shadow DOM traversal. - - `src/domParser.ts`: Wrapper around BuildDomTreeEngine. Produces `AccessibilityNode[]` with highlightIndex, xpath, isTopElement, isInteractive. Also defines the `AccessibilityNode` TypeScript interface. - - `src/serializer.ts`: LLM-optimized `[index]text />` compact text format with token dedup. - - `src/cssSelector.ts`: `enhancedCssSelectorForElement()` — XPath-to-CSS + class names + safe attributes. - - `src/elementHash.ts`: SHA-256 element identity for cross-mutation element matching. - - `src/stability.ts`: Element position stability polling (getBoundingClientRect until delta < 2px). - - `src/iframeBus.ts`: `postMessage` relay between subframes and main frame. - - `src/interaction.ts`: Framework-safe input simulation, sendKeys, scroll variants, dropdown handling. - - `src/bridge.ts`: JS-to-Kotlin bridge (`@JavascriptInterface` wrapper). -- `app/`: Sample/demo application module. -- `docs/`: User-facing documentation (`getting-started.md`, `agent-perception.md`, `best-practices.md`). +## Architecture boundaries -## AgentAction Types -- `Click(agentId)`, `LongPress(agentId, durationMs)`, `InputText(agentId, text, clearFirst)` -- `SelectOption(agentId, value)`, `Scroll(direction: ScrollDirection, amount: Float = 0.5f)`, `Navigate(url)` -- `GoBack`, `GoForward`, `Refresh`, `Wait(durationMs)` -- `SendKeys(keys)` — Keyboard shortcuts (e.g., `"Control+A"`, `"Enter"`) -- `ScrollToPercent(yPercent, agentId?)` — Scroll to percentage position -- `ScrollToText(text, nth)` — Find visible text and scroll to it -- `ScrollToTop(agentId?)`, `ScrollToBottom(agentId?)` — Scroll to extremes -- `PreviousPage(agentId?)`, `NextPage(agentId?)` — Scroll by viewport height -- `GetDropdownOptions(agentId)` — Enumerate ` - - - - diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ConcurrencyStressTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ConcurrencyStressTest.kt deleted file mode 100644 index 64db8a2..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ConcurrencyStressTest.kt +++ /dev/null @@ -1,49 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.* -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith -import java.util.concurrent.atomic.AtomicInteger - -@RunWith(AndroidJUnit4::class) -class ConcurrencyStressTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testConcurrentCaptureState() = runBlocking { - serverRule.enqueueHtml("") - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val successCount = AtomicInteger(0) - val jobs = List(50) { - launch(Dispatchers.Default) { - val result = controller.captureState() - if (result is AgentResult.Success) { - successCount.incrementAndGet() - } - } - } - - jobs.joinAll() - assertEquals(50, successCount.get()) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ControllerCoreTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ControllerCoreTest.kt deleted file mode 100644 index 384b899..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ControllerCoreTest.kt +++ /dev/null @@ -1,88 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class ControllerCoreTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testCaptureState() = runBlocking { - val html = """ - - - - - - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - // Navigate to the test page - val navResult = controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - assertTrue(navResult is AgentResult.Success) - - // Capture state - val stateResult = controller.captureState() - assertTrue(stateResult is AgentResult.Success) - val state = (stateResult as AgentResult.Success).data - - assertTrue(state.accessibilityTree.contains("Click Me")) - assertEquals(serverRule.url("/"), state.url) - assertNotNull(state.screenshotBase64) - } - - @Test - fun testClickAction() = runBlocking { - val html = """ - - - - - - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val state = (controller.captureState() as AgentResult.Success).data - // Find the button agentId (it should be "1" since it's the first interactive element) - val agentId = "1" - - val clickResult = controller.executeAction(AgentAction.Click(agentId)) - assertTrue(clickResult is AgentResult.Success) - - // Wait a bit and check state again - Thread.sleep(500) - val newState = (controller.captureState() as AgentResult.Success).data - assertTrue(newState.accessibilityTree.contains("Clicked")) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/CoordinateMappingTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/CoordinateMappingTest.kt deleted file mode 100644 index 7383983..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/CoordinateMappingTest.kt +++ /dev/null @@ -1,57 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.json.JSONObject -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class CoordinateMappingTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testCoordinateAccuracy() = runBlocking { - val html = """ - - - - - - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val state = (controller.captureState() as AgentResult.Success).data - val btn = JSONObject(state.accessibilityTree).getJSONArray("tree").getJSONObject(0) - - assertEquals("100", btn.getJSONObject("bounds").getString("left")) - assertEquals("200", btn.getJSONObject("bounds").getString("top")) - - // Click it - controller.executeAction(AgentAction.Click("1")) - - Thread.sleep(500) - val newState = (controller.captureState() as AgentResult.Success).data - assertTrue(newState.accessibilityTree.contains("Clicked")) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/DomEdgeCaseTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/DomEdgeCaseTest.kt deleted file mode 100644 index 3b50814..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/DomEdgeCaseTest.kt +++ /dev/null @@ -1,88 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.json.JSONObject -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class DomEdgeCaseTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testShadowDom() = runBlocking { - val html = """ - - - -
- - - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val state = (controller.captureState() as AgentResult.Success).data - assertTrue(state.accessibilityTree.contains("Shadow Button")) - } - - @Test - fun testOcclusionDetection() = runBlocking { - val html = """ - - - - -
Overlay
- - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val state = (controller.captureState() as AgentResult.Success).data - val tree = JSONObject(state.accessibilityTree).getJSONArray("tree") - - var targetOccluded = false - for (i in 0 until tree.length()) { - val node = tree.getJSONObject(i) - if (node.getString("text") == "Target") { - targetOccluded = node.getBoolean("occluded") - } - } - - assertTrue("Target should be occluded", targetOccluded) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/EnhancedFeaturesTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/EnhancedFeaturesTest.kt deleted file mode 100644 index 7c935fe..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/EnhancedFeaturesTest.kt +++ /dev/null @@ -1,300 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class EnhancedFeaturesTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - private fun createController(config: AgenticWebViewConfig = AgenticWebViewConfig(enableDebugLogging = true)): AgenticWebController { - val controller = AgenticWebController(config) - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - return controller - } - - private suspend fun navigateAndLoad(controller: AgenticWebController, html: String): AgentState { - serverRule.enqueueHtml(html) - val navResult = controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - assertTrue("Navigation should succeed", navResult is AgentResult.Success) - return (controller.captureState() as AgentResult.Success).data - } - - @Test - fun testCaptureStateIncludesSelectorMap() = runBlocking { - val html = """ - - - - - """.trimIndent() - - val controller = createController() - val state = navigateAndLoad(controller, html) - - assertNotNull("selectorMap should be present", state.selectorMap) - assertTrue("selectorMap should not be empty", state.selectorMap!!.isNotEmpty()) - } - - @Test - fun testCaptureStateIncludesCompactTree() = runBlocking { - val html = """ - - - - - """.trimIndent() - - val controller = createController() - val state = navigateAndLoad(controller, html) - - assertNotNull("compactTree should be present", state.compactTree) - assertTrue("compactTree should contain button reference", state.compactTree!!.contains("button")) - } - - @Test - fun testSendKeysAction() = runBlocking { - val html = """ - - - - - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.SendKeys("Enter")) - assertTrue("SendKeys should succeed", result is AgentResult.Success) - } - - @Test - fun testScrollToTopAction() = runBlocking { - val html = """ - - -
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.ScrollToTop()) - assertTrue("ScrollToTop should succeed", result is AgentResult.Success) - } - - @Test - fun testScrollToBottomAction() = runBlocking { - val html = """ - - -
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.ScrollToBottom()) - assertTrue("ScrollToBottom should succeed", result is AgentResult.Success) - } - - @Test - fun testScrollToPercentAction() = runBlocking { - val html = """ - - -
Content
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.ScrollToPercent(50f)) - assertTrue("ScrollToPercent should succeed", result is AgentResult.Success) - } - - @Test - fun testScrollToTextAction() = runBlocking { - val html = """ - - -
Tall content
-

Find this text

- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.ScrollToText("Find this text")) - assertTrue("ScrollToText should succeed", result is AgentResult.Success) - } - - @Test - fun testPreviousPageAction() = runBlocking { - val html = """ - - -
Content
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.PreviousPage()) - assertTrue("PreviousPage should succeed", result is AgentResult.Success) - } - - @Test - fun testNextPageAction() = runBlocking { - val html = """ - - -
Content
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.NextPage()) - assertTrue("NextPage should succeed", result is AgentResult.Success) - } - - @Test - fun testSelectDropdownOptionAction() = runBlocking { - val html = """ - - - - - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.SelectDropdownOption("0", "Beta")) - assertTrue("SelectDropdownOption should succeed", result is AgentResult.Success) - } - - @Test - fun testGetDropdownOptionsPublicMethod() = runBlocking { - val html = """ - - - - - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.getDropdownOptions("0") - assertTrue("getDropdownOptions should succeed", result is AgentResult.Success) - val options = (result as AgentResult.Success).data - assertTrue("Should have at least 2 options", options.size >= 2) - assertEquals("x", options[0].value) - assertEquals("X-Ray", options[0].text) - } - - @Test - fun testScrollDirectionEnum() { - val directions = ScrollDirection.entries - assertEquals(4, directions.size) - assertTrue(directions.contains(ScrollDirection.UP)) - assertTrue(directions.contains(ScrollDirection.DOWN)) - assertTrue(directions.contains(ScrollDirection.LEFT)) - assertTrue(directions.contains(ScrollDirection.RIGHT)) - } - - @Test - fun testConfigViewportExpansion() { - val config = AgenticWebViewConfig(viewportExpansion = 100) - assertEquals(100, config.viewportExpansion) - } - - @Test - fun testConfigElementStabilityTimeout() { - val config = AgenticWebViewConfig(elementStabilityTimeoutMs = 2000) - assertEquals(2000L, config.elementStabilityTimeoutMs) - } - - @Test - fun testConfigAntiDetection() { - val config = AgenticWebViewConfig(enableAntiDetection = false) - assertFalse(config.enableAntiDetection) - } - - @Test - fun testConfigBuilderNewOptions() { - val config = AgenticWebViewConfig.Builder() - .setViewportExpansion(200) - .setElementStabilityTimeoutMs(1500) - .setEnableAntiDetection(false) - .setIncludeAttributes(listOf("id", "class")) - .setDeniedHosts(setOf("evil.com")) - .setHomeUrl("https://safe.com") - .build() - - assertEquals(200, config.viewportExpansion) - assertEquals(1500L, config.elementStabilityTimeoutMs) - assertFalse(config.enableAntiDetection) - assertEquals(listOf("id", "class"), config.includeAttributes) - assertEquals(setOf("evil.com"), config.deniedHosts) - assertEquals("https://safe.com", config.homeUrl) - } - - @Test - fun testScrollToTopWithAgentId() = runBlocking { - val html = """ - - -
- - """.trimIndent() - - val controller = createController() - navigateAndLoad(controller, html) - - val result = controller.executeAction(AgentAction.ScrollToTop(agentId = null)) - assertTrue("ScrollToTop with null agentId should succeed", result is AgentResult.Success) - } - - @Test - fun testDoneAction() = runBlocking { - val controller = createController() - val result = controller.executeAction(AgentAction.Done("Task complete", true)) - assertTrue("Done action should succeed", result is AgentResult.Success) - } - - @Test - fun testDeniedHostsConfig() { - val config = AgenticWebViewConfig(deniedHosts = setOf("ads.com", "tracker.io")) - assertEquals(setOf("ads.com", "tracker.io"), config.deniedHosts) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ErrorRecoveryTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ErrorRecoveryTest.kt deleted file mode 100644 index 25d7ea2..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/ErrorRecoveryTest.kt +++ /dev/null @@ -1,56 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class ErrorRecoveryTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testNavigationTimeout() = runBlocking { - // Enqueue a response that never finishes or takes too long - serverRule.enqueueDelayedHtml("Delayed", 5000) - - val config = AgenticWebViewConfig(pageSettleTimeoutMs = 1000) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - val result = controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - assertTrue(result is AgentResult.Error) - assertTrue((result as AgentResult.Error).error is AgentError.Timeout) - } - - @Test - fun testElementNotFound() = runBlocking { - serverRule.enqueueHtml("Empty") - - val config = AgenticWebViewConfig() - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val result = controller.executeAction(AgentAction.Click("non-existent")) - assertTrue(result is AgentResult.Error) - assertTrue((result as AgentResult.Error).error is AgentError.ElementNotFound) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/InputCompatibilityTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/InputCompatibilityTest.kt deleted file mode 100644 index 70a7304..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/InputCompatibilityTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class InputCompatibilityTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testFrameworkSafeInput() = runBlocking { - val html = """ - - - - - - - """.trimIndent() - serverRule.enqueueHtml(html) - - val config = AgenticWebViewConfig(enableDebugLogging = true) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val result = controller.executeAction(AgentAction.InputText("1", "Hello SDK")) - assertTrue(result is AgentResult.Success) - - Thread.sleep(500) - val state = (controller.captureState() as AgentResult.Success).data - assertTrue(state.accessibilityTree.contains("Hello SDK")) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/MockWebServerRule.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/MockWebServerRule.kt deleted file mode 100644 index 8d5a206..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/MockWebServerRule.kt +++ /dev/null @@ -1,49 +0,0 @@ -package dev.shantoislam.agenticwebview - -import mockwebserver3.MockResponse -import mockwebserver3.MockWebServer -import org.junit.rules.ExternalResource -import java.util.concurrent.TimeUnit - -class MockWebServerRule : ExternalResource() { - val server = MockWebServer() - - override fun before() { - server.start() - } - - override fun after() { - server.close() - } - - fun url(path: String): String { - return server.url(path).toString() - } - - fun enqueueHtml(html: String) { - server.enqueue( - MockResponse.Builder() - .body(html) - .addHeader("Content-Type", "text/html") - .build() - ) - } - - fun enqueueDelayedHtml(html: String, delayMs: Long) { - server.enqueue( - MockResponse.Builder() - .body(html) - .addHeader("Content-Type", "text/html") - .bodyDelay(delayMs, TimeUnit.MILLISECONDS) - .build() - ) - } - - fun enqueueError(code: Int) { - server.enqueue( - MockResponse.Builder() - .code(code) - .build() - ) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/SecurityTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/SecurityTest.kt deleted file mode 100644 index 59e77bb..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/SecurityTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class SecurityTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testBlockedHost() = runBlocking { - serverRule.enqueueHtml("Host") - - val config = AgenticWebViewConfig(allowedHosts = setOf("trusted.com")) - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - // Navigate to a non-trusted host (localhost from MockWebServer) - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - // The URL should not be loaded, or at least the state should reflect failure if we had better tracking. - // For now we check if it's still IDLE or LOADING if it was blocked. - // Actually, shouldOverrideUrlLoading blocks it. - - val state = (controller.captureState() as AgentResult.Success).data - assertNotEquals(serverRule.url("/"), state.url) - } -} diff --git a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/StabilityTest.kt b/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/StabilityTest.kt deleted file mode 100644 index 71007fe..0000000 --- a/agentic-webview/src/androidTest/java/dev/shantoislam/agenticwebview/StabilityTest.kt +++ /dev/null @@ -1,77 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.ui.test.junit4.v2.createComposeRule -import androidx.test.ext.junit.runners.AndroidJUnit4 -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.runBlocking -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(AndroidJUnit4::class) -class StabilityTest { - - @get:Rule - val composeTestRule = createComposeRule() - - @get:Rule - val serverRule = MockWebServerRule() - - @Test - fun testIdStabilityAcrossMutations() = runBlocking { - serverRule.enqueueHtml(""" - - - -
- - - """.trimIndent()) - - val config = AgenticWebViewConfig() - val controller = AgenticWebController(config) - - composeTestRule.setContent { - AgenticWebViewComposable(controller = controller, config = config) - } - - controller.executeAction(AgentAction.Navigate(serverRule.url("/"))) - - val state1 = (controller.captureState() as AgentResult.Success).data - val btnId = state1.accessibilityTree.let { tree -> - // Simple parsing of the tree JSON string (since tree is a JSON string in AgentState) - // In a real test we'd use a JSON library - val regex = """"id":"(\d+)","tag":"BUTTON"""".toRegex() - regex.find(tree)?.groupValues?.get(1) - } - - assertNotNull("Button ID should not be null", btnId) - - // Trigger mutation - controller.executeAction(AgentAction.Wait(100)) - // We'll use evalJs directly to trigger a DOM change without re-navigating - // AgenticWebController.evalJs is private, so we might need to use a Navigate to a slightly different page - // OR better, we trust the MutationObserver in the script. - - // Let's use evaluateJavascript via reflection or just navigate to a page that appends an element. - serverRule.enqueueHtml(""" - - - -
New Element
- - - """.trimIndent()) - controller.executeAction(AgentAction.Navigate(serverRule.url("/update"))) - - val state2 = (controller.captureState() as AgentResult.Success).data - val btnIdAfter = state2.accessibilityTree.let { tree -> - val regex = """"id":"$btnId","tag":"BUTTON"""".toRegex() - regex.find(tree)?.groupValues?.get(0) - } - - assertNotNull("Button ID should be the same after mutation/re-render if it's the same element", btnIdAfter) - } -} diff --git a/agentic-webview/src/debug/AndroidManifest.xml b/agentic-webview/src/debug/AndroidManifest.xml deleted file mode 100644 index a8ac6f9..0000000 --- a/agentic-webview/src/debug/AndroidManifest.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - diff --git a/agentic-webview/src/debug/java/dev/shantoislam/agenticwebview/TestActivity.kt b/agentic-webview/src/debug/java/dev/shantoislam/agenticwebview/TestActivity.kt deleted file mode 100644 index da70fac..0000000 --- a/agentic-webview/src/debug/java/dev/shantoislam/agenticwebview/TestActivity.kt +++ /dev/null @@ -1,14 +0,0 @@ -package dev.shantoislam.agenticwebview - -import android.os.Bundle -import androidx.activity.ComponentActivity -import androidx.activity.compose.setContent - -class TestActivity : ComponentActivity() { - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - setContent { - // Placeholder for tests to inject content - } - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebController.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebController.kt deleted file mode 100644 index b86e570..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebController.kt +++ /dev/null @@ -1,514 +0,0 @@ -package dev.shantoislam.agenticwebview - -import android.os.* -import android.view.MotionEvent -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.internal.JsEvaluator -import dev.shantoislam.agenticwebview.internal.ScreenshotCapture -import dev.shantoislam.agenticwebview.internal.SdkLogger -import dev.shantoislam.agenticwebview.models.* -import kotlinx.coroutines.* -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.json.JSONArray -import org.json.JSONObject - -class AgenticWebController( - private val config: AgenticWebViewConfig = AgenticWebViewConfig() -) { - private val logger = SdkLogger(config.enableDebugLogging) - private var webView: AgenticWebView? = null - private val mutex = Mutex() - private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob()) - private var settlementJob: Job? = null - - private val pendingPromises = mutableMapOf>() - - private val jsEvaluator = JsEvaluator( - webViewProvider = { webView }, - timeoutMs = config.jsEvaluationTimeoutMs, - logger = logger - ) - - private val screenshotCapture = ScreenshotCapture( - webViewProvider = { webView }, - quality = config.screenshotQuality, - maxDimension = config.screenshotMaxDimension, - logger = logger - ) - - var lastDropdownOptions: String? = null - private set - - private val occludedMap = mutableMapOf() - - private val _state = MutableStateFlow(null) - val state: StateFlow = _state.asStateFlow() - - private val _loadingProgress = MutableStateFlow(0) - val loadingProgress: StateFlow = _loadingProgress.asStateFlow() - - // ─── Attach / Detach ────────────────────────────────────────────── - - fun attach(webView: AgenticWebView) { - this.webView = webView - webView.listener = object : AgenticWebView.AgenticWebViewListener { - override fun onStateChanged(state: PageLifecycleState) { - logger.d("Controller", "State changed: $state") - if (state == PageLifecycleState.LOADING || - state == PageLifecycleState.CRASHED || - state == PageLifecycleState.ERROR) { - cancelAllPendingPromises("Page lifecycle changed to $state") - } - } - - override fun onProgressChanged(progress: Int) { - _loadingProgress.value = progress - if (progress == 100) { - startSettlementTimer() - } - } - - override fun onDomMutated(json: String) { - startSettlementTimer() - // Only capture state if page is in a stable lifecycle phase - val currentPhase = this@AgenticWebController.webView?.pageLifecycleState - if (currentPhase == PageLifecycleState.COMPLETE || - currentPhase == PageLifecycleState.INTERACTIVE) { - scope.launch { - val currentState = captureState() - if (currentState is AgentResult.Success) { - _state.value = currentState.data - } - } - } - } - - override fun onPromiseResolved(promiseId: String, result: String) { - this@AgenticWebController.onPromiseResolved(promiseId, result) - } - - override fun onCrash(didRecover: Boolean) { - logger.e("Controller", "WebView crashed. Recovery attempted: $didRecover") - _state.value = null - } - } - } - - fun detach() { - webView?.listener = null - webView = null - settlementJob?.cancel() - } - - private fun startSettlementTimer() { - settlementJob?.cancel() - settlementJob = scope.launch { - delay(config.pageSettleDebounceMs) - webView?.updatePageState(PageLifecycleState.COMPLETE) - } - } - - // ─── JS Evaluation (delegated to JsEvaluator) ────────────────── - - private suspend fun evalJsRaw(script: String): AgentResult = - jsEvaluator.evalRaw(script) - - private suspend fun evalJsJson(script: String): JSONObject? = - jsEvaluator.evalJson(script) - - private suspend fun evalJsBool(script: String): Boolean = - jsEvaluator.evalBool(script) - - private suspend fun evalJsVoid(script: String) = - jsEvaluator.evalVoid(script) - - // ─── State Capture ──────────────────────────────────────────────── - - suspend fun captureState(): AgentResult = mutex.withLock { - val wv = webView ?: return AgentResult.Error(AgentError.PageNotReady(PageLifecycleState.IDLE)) - - var fullCaptureObj = evalJsJson("__AgenticInternal.getFullCapture(${config.maxDomElements})") - if (fullCaptureObj == null) { - val injectionError = wv.ensureEngineAvailable() - if (injectionError.isNotEmpty()) { - return AgentResult.Error(AgentError.JsEvaluationFailed(injectionError)) - } - - var retryDelay = 100L - for (attempt in 1..3) { - fullCaptureObj = evalJsJson("__AgenticInternal.getFullCapture(${config.maxDomElements})") - if (fullCaptureObj != null) break - if (attempt < 3) { - delay(retryDelay) - retryDelay *= 2 - } - } - if (fullCaptureObj == null) { - return AgentResult.Error(AgentError.JsEvaluationFailed("JS engine initialized, but getFullCapture returned null.")) - } - } - - return try { - val viewportObj = evalJsJson("__AgenticInternal.getViewportInfo()") - ?: return AgentResult.Error(AgentError.JsEvaluationFailed("Viewport info returned null")) - - val screenshot = if (config.screenshotEnabled) { - when (val result = captureScreenshot()) { - is AgentResult.Success -> result.data - is AgentResult.Error -> null - } - } else null - - val treeArr = fullCaptureObj.optJSONArray("tree") - occludedMap.clear() - if (treeArr != null) { - for (i in 0 until treeArr.length()) { - val node = treeArr.optJSONObject(i) ?: continue - val id = node.optString("id", "") - if (id.isNotEmpty()) { - occludedMap[id] = node.optBoolean("occluded", false) - } - } - } - - val selectorMapObj = fullCaptureObj.optJSONObject("selectorMap") - val selectorMap = mutableMapOf() - selectorMapObj?.keys()?.forEach { key -> - selectorMap[key] = selectorMapObj.optString(key, "") - } - - val compactTree = fullCaptureObj.optString("compactTree", "").ifBlank { null } - - AgentResult.Success(AgentState( - accessibilityTree = fullCaptureObj.optString("tree", "[]"), - screenshotBase64 = screenshot, - viewportInfo = parseViewportInfo(viewportObj), - url = withContext(Dispatchers.Main) { wv.url } ?: "", - title = withContext(Dispatchers.Main) { wv.title } ?: "", - pageState = withContext(Dispatchers.Main) { wv.pageLifecycleState }, - elementCount = treeArr?.length() ?: 0, - truncated = fullCaptureObj.optBoolean("truncated", false), - selectorMap = selectorMap.ifEmpty { null }, - compactTree = compactTree - )) - } catch (e: Exception) { - logger.e("Controller", "captureState failed", e) - AgentResult.Error(AgentError.JsEvaluationFailed(e.message ?: "Unknown error")) - } - } - - // ─── Action Execution ───────────────────────────────────────────── - - suspend fun executeAction(action: AgentAction): AgentResult = mutex.withLock { - repeat(config.actionRetryCount + 1) { attempt -> - val result = executeActionInternal(action) - if (result is AgentResult.Success) return result - if (result is AgentResult.Error && result.error is AgentError.NoNavigationHistory) return result - if (attempt == config.actionRetryCount) return result - delay(500) - } - return AgentResult.Error(AgentError.Timeout("Action execution", 0)) - } - - private suspend fun executeActionInternal(action: AgentAction): AgentResult { - val wv = webView ?: return AgentResult.Error(AgentError.PageNotReady(PageLifecycleState.IDLE)) - - return when (action) { - is AgentAction.Click -> handleClick(action.agentId) - is AgentAction.LongPress -> handleLongPress(action.agentId, action.durationMs) - is AgentAction.InputText -> handleInputText(action.agentId, action.text, action.clearFirst) - is AgentAction.SelectOption -> handleSelectOption(action.agentId, action.value) - is AgentAction.Scroll -> handleScroll(action.direction, action.amount) - is AgentAction.Navigate -> handleNavigate(action.url) - is AgentAction.GoBack -> { - if (!withContext(Dispatchers.Main) { wv.canGoBack() }) { - return AgentResult.Error(AgentError.NoNavigationHistory("back")) - } - val urlBefore = withContext(Dispatchers.Main) { wv.url } - withContext(Dispatchers.Main) { wv.goBack() } - waitForPageSettlement(wv, urlBefore) - } - is AgentAction.GoForward -> { - if (!withContext(Dispatchers.Main) { wv.canGoForward() }) { - return AgentResult.Error(AgentError.NoNavigationHistory("forward")) - } - val urlBefore = withContext(Dispatchers.Main) { wv.url } - withContext(Dispatchers.Main) { wv.goForward() } - waitForPageSettlement(wv, urlBefore) - } - is AgentAction.Refresh -> { - val urlBefore = withContext(Dispatchers.Main) { wv.url } - withContext(Dispatchers.Main) { wv.reload() } - waitForPageSettlement(wv, urlBefore) - } - is AgentAction.Wait -> { delay(action.durationMs); AgentResult.Success(Unit) } - is AgentAction.SendKeys -> handleSendKeys(action.keys) - is AgentAction.ScrollToPercent -> handleScrollToPercent(action.yPercent, action.agentId) - is AgentAction.ScrollToText -> handleScrollToText(action.text, action.nth) - is AgentAction.ScrollToTop -> handleScrollToTop(action.agentId) - is AgentAction.ScrollToBottom -> handleScrollToBottom(action.agentId) - is AgentAction.PreviousPage -> handlePreviousPage(action.agentId) - is AgentAction.NextPage -> handleNextPage(action.agentId) - is AgentAction.GetDropdownOptions -> handleGetDropdownOptions(action.agentId) - is AgentAction.SelectDropdownOption -> handleSelectDropdownOption(action.agentId, action.text) - is AgentAction.Done -> AgentResult.Success(Unit) - } - } - - // ─── Action Handlers ────────────────────────────────────────────── - - private suspend fun handleClick(agentId: String): AgentResult { - val isFile = evalJsBool("__AgenticInternal.isFileUploader('$agentId')") - if (isFile) return AgentResult.Error(AgentError.FileUploaderDetected(agentId)) - - if (occludedMap[agentId] == true) { - logger.w("Controller", "Element $agentId is occluded, proceeding with click anyway") - } - - val coords = getElementCoords(agentId) ?: return AgentResult.Error(AgentError.ElementNotFound(agentId)) - withContext(Dispatchers.Main) { dispatchTouch(coords.first, coords.second) } - return AgentResult.Success(Unit) - } - - private suspend fun handleLongPress(agentId: String, durationMs: Long): AgentResult { - val coords = getElementCoords(agentId) ?: return AgentResult.Error(AgentError.ElementNotFound(agentId)) - withContext(Dispatchers.Main) { dispatchTouch(coords.first, coords.second, durationMs) } - return AgentResult.Success(Unit) - } - - private suspend fun handleInputText(agentId: String, text: String, clearFirst: Boolean): AgentResult { - evalJsBool("__AgenticInternal.waitForStability('$agentId', ${config.elementStabilityTimeoutMs})") - val coords = getElementCoords(agentId) ?: return AgentResult.Error(AgentError.ElementNotFound(agentId)) - withContext(Dispatchers.Main) { dispatchTouch(coords.first, coords.second) } - delay(200) - val textJson = JSONObject.quote(text) - val success = evalJsBool("__AgenticInternal.setInputValue('$agentId', $textJson)") - return if (success) AgentResult.Success(Unit) - else AgentResult.Error(AgentError.JsEvaluationFailed("Failed to set input value")) - } - - private suspend fun handleSelectOption(agentId: String, value: String): AgentResult { - val valueJson = JSONObject.quote(value) - val success = evalJsBool("__AgenticInternal.setSelectOption('$agentId', $valueJson)") - return if (success) AgentResult.Success(Unit) - else AgentResult.Error(AgentError.JsEvaluationFailed("Failed to set select option")) - } - - private suspend fun handleSendKeys(keys: String): AgentResult { - val keysJson = JSONObject.quote(keys) - val success = evalJsBool("__AgenticInternal.sendKeys($keysJson)") - return if (success) AgentResult.Success(Unit) - else AgentResult.Error(AgentError.JsEvaluationFailed("Failed to send keys")) - } - - private suspend fun handleScrollToPercent(yPercent: Float, agentId: String?): AgentResult { - val agentIdArg = if (agentId != null) "'$agentId'" else "undefined" - evalJsVoid("__AgenticInternal.scrollToPercent($yPercent, $agentIdArg)") - return AgentResult.Success(Unit) - } - - private suspend fun handleScrollToText(text: String, nth: Int): AgentResult { - val textJson = JSONObject.quote(text) - val success = evalJsBool("__AgenticInternal.scrollToText($textJson, $nth)") - return if (success) AgentResult.Success(Unit) - else AgentResult.Error(AgentError.ElementNotFound("text:$text")) - } - - private suspend fun handleScrollToTop(agentId: String?): AgentResult { - val agentIdArg = if (agentId != null) "'$agentId'" else "undefined" - evalJsVoid("__AgenticInternal.scrollToTop($agentIdArg)") - return AgentResult.Success(Unit) - } - - private suspend fun handleScrollToBottom(agentId: String?): AgentResult { - val agentIdArg = if (agentId != null) "'$agentId'" else "undefined" - evalJsVoid("__AgenticInternal.scrollToBottom($agentIdArg)") - return AgentResult.Success(Unit) - } - - private suspend fun handlePreviousPage(agentId: String?): AgentResult { - val agentIdArg = if (agentId != null) "'$agentId'" else "undefined" - evalJsVoid("__AgenticInternal.previousPage($agentIdArg)") - return AgentResult.Success(Unit) - } - - private suspend fun handleNextPage(agentId: String?): AgentResult { - val agentIdArg = if (agentId != null) "'$agentId'" else "undefined" - evalJsVoid("__AgenticInternal.nextPage($agentIdArg)") - return AgentResult.Success(Unit) - } - - private suspend fun handleGetDropdownOptions(agentId: String): AgentResult { - val result = evalJsRaw("__AgenticInternal.getDropdownOptions('$agentId')") - val json = when (result) { - is AgentResult.Success -> result.data.ifBlank { "[]" } - is AgentResult.Error -> "[]" - } - lastDropdownOptions = json - logger.d("Controller", "Dropdown options: $lastDropdownOptions") - return AgentResult.Success(Unit) - } - - suspend fun canGoBack(): Boolean = withContext(Dispatchers.Main) { webView?.canGoBack() == true } - - suspend fun canGoForward(): Boolean = withContext(Dispatchers.Main) { webView?.canGoForward() == true } - - suspend fun getDropdownOptions(agentId: String): AgentResult> = mutex.withLock { - return try { - val json = when (val result = evalJsRaw("__AgenticInternal.getDropdownOptions('$agentId')")) { - is AgentResult.Success -> result.data.ifBlank { "[]" } - is AgentResult.Error -> "[]" - } - lastDropdownOptions = json - val arr = JSONArray(lastDropdownOptions) - val options = mutableListOf() - for (i in 0 until arr.length()) { - val obj = arr.optJSONObject(i) ?: continue - options.add(DropdownOption( - value = obj.optString("value", ""), - text = obj.optString("text", ""), - index = obj.optInt("index", i) - )) - } - AgentResult.Success(options) - } catch (e: Exception) { - AgentResult.Error(AgentError.JsEvaluationFailed(e.message ?: "Failed to parse dropdown options")) - } - } - - private suspend fun handleSelectDropdownOption(agentId: String, text: String): AgentResult { - val textJson = JSONObject.quote(text) - val success = evalJsBool("__AgenticInternal.selectDropdownOption('$agentId', $textJson)") - return if (success) AgentResult.Success(Unit) - else AgentResult.Error(AgentError.JsEvaluationFailed("Failed to select dropdown option")) - } - - private suspend fun handleScroll(direction: ScrollDirection, amount: Float): AgentResult { - val script = when (direction) { - ScrollDirection.UP -> "window.scrollBy(0, -window.innerHeight * $amount)" - ScrollDirection.DOWN -> "window.scrollBy(0, window.innerHeight * $amount)" - ScrollDirection.LEFT -> "window.scrollBy(-window.innerWidth * $amount, 0)" - ScrollDirection.RIGHT -> "window.scrollBy(window.innerWidth * $amount, 0)" - } - evalJsVoid(script) - return AgentResult.Success(Unit) - } - - private suspend fun waitForPageSettlement(wv: AgenticWebView, urlForError: String?): AgentResult { - val settled = withTimeoutOrNull(config.pageSettleTimeoutMs) { - while (true) { - val state = withContext(Dispatchers.Main) { wv.pageLifecycleState } - if (state == PageLifecycleState.COMPLETE) break - if (state == PageLifecycleState.ERROR) { - val httpCode = withContext(Dispatchers.Main) { wv.lastNavigationHttpError } - return@withTimeoutOrNull AgentResult.Error( - AgentError.NavigationFailed(urlForError ?: wv.url ?: "", httpCode) - ) - } - delay(100) - } - AgentResult.Success(Unit) - } - if (settled != null) return settled - - val httpCode = withContext(Dispatchers.Main) { wv.lastNavigationHttpError } - return if (httpCode != null && httpCode >= 400) { - AgentResult.Error(AgentError.NavigationFailed(urlForError ?: wv.url ?: "", httpCode)) - } else { - AgentResult.Error(AgentError.Timeout("Navigation", config.pageSettleTimeoutMs)) - } - } - - private suspend fun handleNavigate(url: String): AgentResult { - val wv = webView ?: return AgentResult.Error(AgentError.PageNotReady(PageLifecycleState.IDLE)) - withContext(Dispatchers.Main) { wv.loadUrl(url) } - return waitForPageSettlement(wv, url) - } - - // ─── Element Coordinates ────────────────────────────────────────── - - private suspend fun getElementCoords(agentId: String): Pair? { - val promiseId = java.util.UUID.randomUUID().toString() - evalJsVoid("__AgenticInternal.scrollIntoView('$agentId', '$promiseId')") - - val deferred = CompletableDeferred() - pendingPromises[promiseId] = deferred - - return try { - val result = withTimeout(5000) { deferred.await() } - if (result != "true") return null - - val json = evalJsJson("__AgenticInternal.getElementCenter('$agentId')") - ?: return null - val x = json.optDouble("x", Double.NaN) - val y = json.optDouble("y", Double.NaN) - if (x.isNaN() || y.isNaN()) return null - Pair(x.toFloat(), y.toFloat()) - } catch (e: Exception) { - logger.e("Controller", "Failed to resolve scroll promise", e) - null - } finally { - pendingPromises.remove(promiseId) - } - } - - private fun onPromiseResolved(promiseId: String, result: String) { - pendingPromises[promiseId]?.complete(result) - } - - // ─── Touch Dispatch ─────────────────────────────────────────────── - - private suspend fun dispatchTouch(x: Float, y: Float, durationMs: Long = 0) = withContext(Dispatchers.Main) { - val wv = webView ?: return@withContext - val downTime = SystemClock.uptimeMillis() - val downEvent = MotionEvent.obtain(downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0) - wv.dispatchTouchEvent(downEvent) - - if (durationMs > 0) { delay(durationMs) } - - val upTime = SystemClock.uptimeMillis() - val upEvent = MotionEvent.obtain(downTime, upTime, MotionEvent.ACTION_UP, x, y, 0) - wv.dispatchTouchEvent(upEvent) - } - - // ─── Screenshot (delegated to ScreenshotCapture) ──────────────── - - private suspend fun captureScreenshot(): AgentResult = - screenshotCapture.capture() - - // ─── Utilities ──────────────────────────────────────────────────── - - private fun parseViewportInfo(obj: JSONObject): ViewportInfo { - return ViewportInfo( - devicePixelRatio = obj.optDouble("devicePixelRatio", 1.0), - visualViewportScale = obj.optDouble("visualViewportScale", 1.0), - scrollX = obj.optInt("scrollX", 0), - scrollY = obj.optInt("scrollY", 0), - viewportWidth = obj.optInt("viewportWidth", 0), - viewportHeight = obj.optInt("viewportHeight", 0) - ) - } - - private fun cancelAllPendingPromises(reason: String) { - if (pendingPromises.isEmpty()) return - logger.w("Controller", "Cancelling all pending promises: $reason") - val promises = HashMap(pendingPromises) - pendingPromises.clear() - for ((_, deferred) in promises) { - deferred.complete("error:page_transition:$reason") - } - } - - fun destroy() { - scope.cancel() - screenshotCapture.destroy() - cancelAllPendingPromises("Controller destroyed") - } - - fun pauseTimers() { webView?.pauseTimers() } - fun resumeTimers() { webView?.resumeTimers() } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebView.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebView.kt deleted file mode 100644 index adfb1b3..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebView.kt +++ /dev/null @@ -1,329 +0,0 @@ -package dev.shantoislam.agenticwebview - -import android.annotation.SuppressLint -import android.content.Context -import android.graphics.Bitmap -import android.os.Handler -import android.os.Looper -import android.os.Message -import android.util.AttributeSet -import android.webkit.* -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.internal.SdkLogger -import dev.shantoislam.agenticwebview.models.PageLifecycleState -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withContext -import kotlin.coroutines.resume - -@SuppressLint("SetJavaScriptEnabled") -class AgenticWebView @JvmOverloads constructor( - context: Context, - private val config: AgenticWebViewConfig = AgenticWebViewConfig(), - attrs: AttributeSet? = null, - defStyleAttr: Int = 0 -) : WebView(context, attrs, defStyleAttr) { - - private val logger = SdkLogger(config.enableDebugLogging) - private var scriptCache: String? = null - var pageLifecycleState: PageLifecycleState = PageLifecycleState.IDLE - private set - - @Volatile - private var currentSessionToken: String = "" - - @Volatile - var isScriptInjected: Boolean = false - private set - - @Volatile - var lastNavigationHttpError: Int? = null - internal set - - var listener: AgenticWebViewListener? = null - - companion object { - private var isDataDirSet = false - - fun init() {} - - fun getVersion(): String = "0.2.1" - } - - init { - setupSettings() - setupClients() - addJavascriptInterface(JsBridge(), "AgenticBridge") - } - - private fun setupSettings() { - settings.apply { - javaScriptEnabled = true - domStorageEnabled = true - allowFileAccess = false - allowContentAccess = false - mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW - userAgentString = config.userAgent ?: userAgentString - } - if (!isDataDirSet) { - try { - setDataDirectorySuffix("agentic_webview") - isDataDirSet = true - } catch (e: Exception) { - // Ignore if already set - } - } - } - - private fun setupClients() { - webViewClient = object : WebViewClient() { - override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { - currentSessionToken = java.util.UUID.randomUUID().toString() - isScriptInjected = false - lastNavigationHttpError = null - pageLifecycleState = PageLifecycleState.LOADING - listener?.onStateChanged(pageLifecycleState) - - if (config.enableAntiDetection) { - view?.evaluateJavascript(""" - (function() { - try { Object.defineProperty(navigator, 'webdriver', { get: function() { return undefined; } }); } catch(e) {} - try { window.chrome = { runtime: {} }; } catch(e) {} - })(); - """.trimIndent(), null) - } - } - - override fun onPageFinished(view: WebView?, url: String?) { - // Do NOT blindly inject script here. We use JIT injection in ensureEngineAvailable. - pageLifecycleState = PageLifecycleState.INTERACTIVE - listener?.onStateChanged(pageLifecycleState) - } - - override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean { - val url = request?.url ?: return false - val scheme = url.scheme - if (scheme != "http" && scheme != "https") { - logger.w("WebView", "Blocking non-http(s) URL: $url") - return true - } - config.deniedHosts?.let { denied -> - val host = url.host ?: return@let - if (denied.any { host.endsWith(it) }) { - logger.w("WebView", "Blocking denied host: $host") - config.homeUrl?.let { view?.loadUrl(it) } - return true - } - } - config.allowedHosts?.let { hosts -> - if (url.host !in hosts) { - logger.w("WebView", "Blocking unauthorized host: ${url.host}") - return true - } - } - return false - } - - override fun onReceivedError(view: WebView?, request: WebResourceRequest?, error: WebResourceError?) { - if (request?.isForMainFrame == true) { - pageLifecycleState = PageLifecycleState.ERROR - listener?.onStateChanged(pageLifecycleState) - } - } - - override fun onReceivedHttpError(view: WebView?, request: WebResourceRequest?, errorResponse: WebResourceResponse?) { - if (request?.isForMainFrame == true) { - lastNavigationHttpError = errorResponse?.statusCode - } - } - - override fun onRenderProcessGone(view: WebView?, detail: RenderProcessGoneDetail?): Boolean { - pageLifecycleState = PageLifecycleState.CRASHED - isScriptInjected = false - listener?.onStateChanged(pageLifecycleState) - logger.e("WebView", "Renderer process gone. Did crash: ${detail?.didCrash()}") - if (detail?.didCrash() == true) { - listener?.onCrash(didRecover = true) - } - return true - } - } - - webChromeClient = object : WebChromeClient() { - override fun onProgressChanged(view: WebView?, newProgress: Int) { - listener?.onProgressChanged(newProgress) - } - - override fun onCreateWindow( - view: WebView?, isDialog: Boolean, isUserGesture: Boolean, resultMsg: Message? - ): Boolean { - val transport = resultMsg?.obj as? WebView.WebViewTransport ?: return false - val newWebView = WebView(context) - newWebView.webViewClient = object : WebViewClient() { - override fun onPageStarted(v: WebView?, url: String?, favicon: Bitmap?) { - url?.let { listener?.onNewTabRequested(it) } - newWebView.destroy() - } - } - transport.webView = newWebView - resultMsg.sendToTarget() - return true - } - - override fun onJsAlert(view: WebView?, url: String?, message: String?, result: JsResult?): Boolean { - logger.i("WebView", "JS Alert: $message") - result?.confirm() - return true - } - - override fun onJsConfirm(view: WebView?, url: String?, message: String?, result: JsResult?): Boolean { - logger.i("WebView", "JS Confirm: $message") - result?.confirm() - return true - } - - override fun onJsPrompt(view: WebView?, url: String?, message: String?, defaultValue: String?, result: JsPromptResult?): Boolean { - logger.i("WebView", "JS Prompt: $message") - result?.confirm() - return true - } - } - } - - /** - * Just-In-Time (JIT) script injection that guarantees the engine is available. - * Uses a robust try-catch wrapper to report exact execution errors. - * Returns an empty string on success, or an error message on failure. - */ - suspend fun ensureEngineAvailable(): String = withContext(Dispatchers.Main) { - if (isScriptInjected) { - // Verify the global is still accessible (page may have navigated) - val verified = suspendCancellableCoroutine { cont -> - evaluateJavascript("typeof window.__AgenticInternal !== 'undefined'") { result -> - cont.resume(result == "true") - } - } - if (verified) return@withContext "" - // Global gone — re-inject - isScriptInjected = false - } - - if (scriptCache == null) { - try { - scriptCache = context.assets.open("agentic_core.min.js").bufferedReader().use { it.readText() } - } catch (e: Exception) { - val errorMsg = "Failed to load script from assets: ${e.message}" - logger.e("WebView", errorMsg, e) - return@withContext errorMsg - } - } - - val wrappedScript = """ - (function() { - try { - if (typeof window.__AgenticInternal !== 'undefined') { - return 'SUCCESS'; - } - ${scriptCache} - - if (typeof window.__AgenticInternal !== 'undefined') { - window.__AgenticInternal.setSessionToken('$currentSessionToken'); - return 'SUCCESS'; - } else { - return 'ERROR: Script executed but window.__AgenticInternal is still undefined.'; - } - } catch (e) { - return 'ERROR: ' + e.message + '\n' + e.stack; - } - })(); - """.trimIndent() - - suspendCancellableCoroutine { cont -> - evaluateJavascript(wrappedScript) { result -> - // The result is a JSON string, so "SUCCESS" becomes "\"SUCCESS\"" - val unquotedResult = if (result != null && result.startsWith("\"") && result.endsWith("\"")) { - result.substring(1, result.length - 1) - .replace("\\\"", "\"") - .replace("\\n", "\n") - } else { - result ?: "ERROR: evaluateJavascript returned null" - } - - if (unquotedResult == "SUCCESS") { - isScriptInjected = true - forwardConfigToEngine() - cont.resume("") - } else { - val errorMsg = "Injection failed: $unquotedResult" - logger.e("WebView", errorMsg) - cont.resume(errorMsg) - } - } - } - } - - fun updatePageState(state: PageLifecycleState) { - pageLifecycleState = state - listener?.onStateChanged(state) - } - - private fun forwardConfigToEngine() { - val configJson = buildString { - append("{") - append("\"viewportExpansion\":${config.viewportExpansion},") - append("\"domMutationThrottleMs\":${config.domMutationThrottleMs},") - append("\"enableAntiDetection\":${config.enableAntiDetection}") - config.includeAttributes?.let { attrs -> - append(",\"includeAttributes\":[") - append(attrs.joinToString(",") { "\"$it\"" }) - append("]") - } - append("}") - } - evaluateJavascript("window.__AgenticInternal && window.__AgenticInternal.configure('$configJson')", null) - } - - inner class JsBridge { - private val mainHandler = Handler(Looper.getMainLooper()) - - @JavascriptInterface - fun onDomUpdate(token: String, json: String) { - if (token != currentSessionToken) { - logger.w("Bridge", "Security alert: Unauthorized token in onDomUpdate") - return - } - mainHandler.post { - listener?.onDomMutated(json) - } - } - - @JavascriptInterface - fun onError(token: String, errorJson: String) { - if (token != currentSessionToken) { - logger.w("Bridge", "Security alert: Unauthorized token in onError") - return - } - logger.e("Bridge", "JS Error: $errorJson") - } - - @JavascriptInterface - fun resolvePromise(token: String, promiseId: String, result: String) { - if (token != currentSessionToken) { - logger.w("Bridge", "Security alert: Unauthorized token in resolvePromise") - return - } - mainHandler.post { - listener?.onPromiseResolved(promiseId, result) - } - } - } - - interface AgenticWebViewListener { - fun onStateChanged(state: PageLifecycleState) - fun onProgressChanged(progress: Int) - fun onDomMutated(json: String) - fun onPromiseResolved(promiseId: String, result: String) - fun onCrash(didRecover: Boolean) - fun onNewTabRequested(url: String) {} - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebViewCompose.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebViewCompose.kt deleted file mode 100644 index 8a4953a..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/AgenticWebViewCompose.kt +++ /dev/null @@ -1,47 +0,0 @@ -package dev.shantoislam.agenticwebview - -import androidx.compose.runtime.Composable -import androidx.compose.runtime.DisposableEffect -import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.platform.LocalLifecycleOwner -import androidx.compose.ui.viewinterop.AndroidView -import androidx.lifecycle.Lifecycle -import androidx.lifecycle.LifecycleEventObserver -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig - -@Composable -fun AgenticWebViewComposable( - controller: AgenticWebController, - modifier: Modifier = Modifier, - config: AgenticWebViewConfig = AgenticWebViewConfig() -) { - val lifecycleOwner = LocalLifecycleOwner.current - - AndroidView( - factory = { context -> - AgenticWebView(context, config).also { webView -> - controller.attach(webView) - } - }, - modifier = modifier, - onRelease = { - controller.detach() - } - ) - - DisposableEffect(lifecycleOwner) { - val observer = LifecycleEventObserver { _, event -> - when (event) { - Lifecycle.Event.ON_PAUSE -> controller.pauseTimers() - Lifecycle.Event.ON_RESUME -> controller.resumeTimers() - Lifecycle.Event.ON_DESTROY -> controller.destroy() - else -> {} - } - } - lifecycleOwner.lifecycle.addObserver(observer) - onDispose { - lifecycleOwner.lifecycle.removeObserver(observer) - } - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/config/AgenticWebViewConfig.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/config/AgenticWebViewConfig.kt deleted file mode 100644 index 0008c36..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/config/AgenticWebViewConfig.kt +++ /dev/null @@ -1,71 +0,0 @@ -package dev.shantoislam.agenticwebview.config - -data class AgenticWebViewConfig( - val jsEvaluationTimeoutMs: Long = 5_000L, - val pageSettleTimeoutMs: Long = 10_000L, - val pageSettleDebounceMs: Long = 500L, - val screenshotEnabled: Boolean = true, - val screenshotQuality: Int = 75, - val screenshotMaxDimension: Int = 1920, - val maxDomElements: Int = 500, - val domMutationThrottleMs: Long = 300L, - val actionRetryCount: Int = 2, - val enableDebugLogging: Boolean = false, - val userAgent: String? = null, - val allowedHosts: Set? = null, - val deniedHosts: Set? = null, - val homeUrl: String? = null, - val viewportExpansion: Int = 0, - val elementStabilityTimeoutMs: Long = 1000L, - val enableAntiDetection: Boolean = true, - val includeAttributes: List? = null -) { - class Builder { - private var jsEvaluationTimeoutMs: Long = 5_000L - private var pageSettleTimeoutMs: Long = 10_000L - private var pageSettleDebounceMs: Long = 500L - private var screenshotEnabled: Boolean = true - private var screenshotQuality: Int = 75 - private var screenshotMaxDimension: Int = 1920 - private var maxDomElements: Int = 500 - private var domMutationThrottleMs: Long = 300L - private var actionRetryCount: Int = 2 - private var enableDebugLogging: Boolean = false - private var userAgent: String? = null - private var allowedHosts: Set? = null - private var deniedHosts: Set? = null - private var homeUrl: String? = null - private var viewportExpansion: Int = 0 - private var elementStabilityTimeoutMs: Long = 1000L - private var enableAntiDetection: Boolean = true - private var includeAttributes: List? = null - - fun setJsEvaluationTimeoutMs(timeout: Long) = apply { this.jsEvaluationTimeoutMs = timeout } - fun setPageSettleTimeoutMs(timeout: Long) = apply { this.pageSettleTimeoutMs = timeout } - fun setPageSettleDebounceMs(debounce: Long) = apply { this.pageSettleDebounceMs = debounce } - fun setScreenshotEnabled(enabled: Boolean) = apply { this.screenshotEnabled = enabled } - fun setScreenshotQuality(quality: Int) = apply { this.screenshotQuality = quality } - fun setScreenshotMaxDimension(dimension: Int) = apply { this.screenshotMaxDimension = dimension } - fun setMaxDomElements(max: Int) = apply { this.maxDomElements = max } - fun setDomMutationThrottleMs(throttle: Long) = apply { this.domMutationThrottleMs = throttle } - fun setActionRetryCount(count: Int) = apply { this.actionRetryCount = count } - fun setEnableDebugLogging(enabled: Boolean) = apply { this.enableDebugLogging = enabled } - fun setUserAgent(userAgent: String?) = apply { this.userAgent = userAgent } - fun setAllowedHosts(hosts: Set?) = apply { this.allowedHosts = hosts } - fun setDeniedHosts(hosts: Set?) = apply { this.deniedHosts = hosts } - fun setHomeUrl(url: String?) = apply { this.homeUrl = url } - fun setViewportExpansion(expansion: Int) = apply { this.viewportExpansion = expansion } - fun setElementStabilityTimeoutMs(timeout: Long) = apply { this.elementStabilityTimeoutMs = timeout } - fun setEnableAntiDetection(enabled: Boolean) = apply { this.enableAntiDetection = enabled } - fun setIncludeAttributes(attrs: List?) = apply { this.includeAttributes = attrs } - - fun build() = AgenticWebViewConfig( - jsEvaluationTimeoutMs, pageSettleTimeoutMs, pageSettleDebounceMs, - screenshotEnabled, screenshotQuality, screenshotMaxDimension, - maxDomElements, domMutationThrottleMs, actionRetryCount, - enableDebugLogging, userAgent, allowedHosts, deniedHosts, homeUrl, - viewportExpansion, elementStabilityTimeoutMs, - enableAntiDetection, includeAttributes - ) - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsEvaluator.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsEvaluator.kt deleted file mode 100644 index 6e7eeac..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsEvaluator.kt +++ /dev/null @@ -1,87 +0,0 @@ -package dev.shantoislam.agenticwebview.internal - -import dev.shantoislam.agenticwebview.AgenticWebView -import dev.shantoislam.agenticwebview.models.AgentError -import dev.shantoislam.agenticwebview.models.AgentResult -import kotlinx.coroutines.* -import org.json.JSONObject -import kotlin.coroutines.resume - -internal class JsEvaluator( - private val webViewProvider: () -> AgenticWebView?, - private val timeoutMs: Long, - private val logger: SdkLogger -) { - suspend fun evalRaw(script: String): AgentResult { - return try { - val result = withContext(Dispatchers.Main) { - withTimeout(timeoutMs) { - suspendCancellableCoroutine { cont -> - val wv = webViewProvider() - if (wv == null) { - cont.resume("") - return@suspendCancellableCoroutine - } - wv.evaluateJavascript(script) { jsResult -> - when { - jsResult == null -> { - logger.d("JsEval", "JS returned null for: ${script.take(80)}") - cont.resume("") - } - jsResult == "null" -> cont.resume("") - jsResult == "undefined" -> cont.resume("") - jsResult.startsWith("\"") && jsResult.endsWith("\"") && jsResult.length >= 2 -> { - try { - val decoded = org.json.JSONTokener(jsResult).nextValue() as String - cont.resume(decoded) - } catch (e: Exception) { - cont.resume( - jsResult.substring(1, jsResult.length - 1) - .replace("\\\\", "\\") - .replace("\\\"", "\"") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\/", "/") - ) - } - } - else -> cont.resume(jsResult) - } - } - } - } - } - AgentResult.Success(result) - } catch (e: TimeoutCancellationException) { - logger.w("JsEval", "JS evaluation timed out: ${script.take(80)}") - AgentResult.Error(AgentError.JsEvaluationTimeout(timeoutMs)) - } catch (e: Exception) { - logger.e("JsEval", "JS evaluation failed: ${script.take(80)}", e) - AgentResult.Error(AgentError.JsEvaluationFailed(e.message ?: "Unknown error")) - } - } - - suspend fun evalJson(script: String): JSONObject? { - return when (val result = evalRaw(script)) { - is AgentResult.Success -> { - if (result.data.isBlank()) null - else try { JSONObject(result.data) } catch (e: Exception) { - logger.w("JsEval", "Failed to parse JSON: ${result.data.take(100)}") - null - } - } - is AgentResult.Error -> null - } - } - - suspend fun evalBool(script: String): Boolean { - return when (val result = evalRaw(script)) { - is AgentResult.Success -> result.data.equals("true", ignoreCase = true) - is AgentResult.Error -> false - } - } - - suspend fun evalVoid(script: String) { - evalRaw(script) - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsUtils.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsUtils.kt deleted file mode 100644 index f566098..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/JsUtils.kt +++ /dev/null @@ -1,16 +0,0 @@ -package dev.shantoislam.agenticwebview.internal - -object JsUtils { - /** - * Escapes a string for use in a JavaScript string literal. - */ - fun escapeJs(input: String?): String { - if (input == null) return "null" - return input.replace("\\", "\\\\") - .replace("'", "\\'") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/ScreenshotCapture.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/ScreenshotCapture.kt deleted file mode 100644 index ef30063..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/ScreenshotCapture.kt +++ /dev/null @@ -1,87 +0,0 @@ -package dev.shantoislam.agenticwebview.internal - -import android.app.Activity -import android.graphics.Bitmap -import android.graphics.Rect -import android.os.Handler -import android.os.HandlerThread -import android.util.Base64 -import android.view.PixelCopy -import androidx.core.graphics.createBitmap -import dev.shantoislam.agenticwebview.AgenticWebView -import dev.shantoislam.agenticwebview.models.AgentError -import dev.shantoislam.agenticwebview.models.AgentResult -import kotlinx.coroutines.suspendCancellableCoroutine -import java.io.ByteArrayOutputStream -import kotlin.coroutines.resume - -internal class ScreenshotCapture( - private val webViewProvider: () -> AgenticWebView?, - private val quality: Int, - private val maxDimension: Int, - private val logger: SdkLogger -) { - private val pixelCopyThread = HandlerThread("PixelCopyThread").apply { start() } - private val pixelCopyHandler = Handler(pixelCopyThread.looper) - private var cachedBitmap: Bitmap? = null - - suspend fun capture(): AgentResult { - val wv = webViewProvider() - ?: return AgentResult.Error(AgentError.ScreenshotFailed("WebView is null")) - if (wv.width <= 0 || wv.height <= 0) - return AgentResult.Error(AgentError.ScreenshotFailed("WebView has zero dimensions")) - - val window = (wv.context as? Activity)?.window - ?: return AgentResult.Error(AgentError.ScreenshotFailed("No Activity window")) - val bitmap = getReusableBitmap(wv.width, wv.height) - - val locationInWindow = IntArray(2) - wv.getLocationInWindow(locationInWindow) - val sourceRect = Rect( - locationInWindow[0], locationInWindow[1], - locationInWindow[0] + wv.width, locationInWindow[1] + wv.height - ) - - return try { - val result = suspendCancellableCoroutine { cont -> - try { - PixelCopy.request(window, sourceRect, bitmap, { cont.resume(it) }, pixelCopyHandler) - } catch (e: Exception) { cont.resume(-1) } - } - if (result == PixelCopy.SUCCESS) { - val scale = minOf(maxDimension.toFloat() / bitmap.width, maxDimension.toFloat() / bitmap.height, 1f) - val finalBitmap = if (scale < 1f) { - val scaledW = (bitmap.width * scale).toInt() - val scaledH = (bitmap.height * scale).toInt() - Bitmap.createScaledBitmap(bitmap, scaledW, scaledH, true) - } else bitmap - - val outputStream = ByteArrayOutputStream() - finalBitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream) - if (finalBitmap !== bitmap) finalBitmap.recycle() - AgentResult.Success(Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP)) - } else { - logger.e("Screenshot", "PixelCopy failed with code: $result") - AgentResult.Error(AgentError.ScreenshotFailed("PixelCopy failed with code $result")) - } - } catch (e: Exception) { - logger.e("Screenshot", "Failed to capture screenshot", e) - AgentResult.Error(AgentError.ScreenshotFailed(e.message ?: "Unknown error")) - } - } - - private fun getReusableBitmap(width: Int, height: Int): Bitmap { - val current = cachedBitmap - if (current != null && current.width == width && current.height == height) return current - current?.recycle() - val newBitmap = createBitmap(width, height) - cachedBitmap = newBitmap - return newBitmap - } - - fun destroy() { - pixelCopyThread.quitSafely() - cachedBitmap?.recycle() - cachedBitmap = null - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/SdkLogger.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/SdkLogger.kt deleted file mode 100644 index ea43b78..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/internal/SdkLogger.kt +++ /dev/null @@ -1,21 +0,0 @@ -package dev.shantoislam.agenticwebview.internal - -import android.util.Log - -internal class SdkLogger(private val enabled: Boolean) { - fun d(tag: String, msg: String) { - if (enabled) Log.d("AgenticSDK:$tag", msg) - } - - fun e(tag: String, msg: String, tr: Throwable? = null) { - if (enabled) Log.e("AgenticSDK:$tag", msg, tr) - } - - fun i(tag: String, msg: String) { - if (enabled) Log.i("AgenticSDK:$tag", msg) - } - - fun w(tag: String, msg: String) { - if (enabled) Log.w("AgenticSDK:$tag", msg) - } -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentAction.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentAction.kt deleted file mode 100644 index 80e583a..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentAction.kt +++ /dev/null @@ -1,27 +0,0 @@ -package dev.shantoislam.agenticwebview.models - -sealed class AgentAction { - data class Click(val agentId: String) : AgentAction() - data class LongPress(val agentId: String, val durationMs: Long = 500) : AgentAction() - data class InputText(val agentId: String, val text: String, val clearFirst: Boolean = true) : AgentAction() - data class SelectOption(val agentId: String, val value: String) : AgentAction() - data class Scroll(val direction: ScrollDirection, val amount: Float = 0.5f) : AgentAction() - data class Navigate(val url: String) : AgentAction() - data object GoBack : AgentAction() - data object GoForward : AgentAction() - data object Refresh : AgentAction() - data class Wait(val durationMs: Long = 1000) : AgentAction() - - data class SendKeys(val keys: String) : AgentAction() - data class ScrollToPercent(val yPercent: Float, val agentId: String? = null) : AgentAction() - data class ScrollToText(val text: String, val nth: Int = 0) : AgentAction() - data class ScrollToTop(val agentId: String? = null) : AgentAction() - data class ScrollToBottom(val agentId: String? = null) : AgentAction() - data class PreviousPage(val agentId: String? = null) : AgentAction() - data class NextPage(val agentId: String? = null) : AgentAction() - data class GetDropdownOptions(val agentId: String) : AgentAction() - data class SelectDropdownOption(val agentId: String, val text: String) : AgentAction() - data class Done(val text: String, val success: Boolean) : AgentAction() -} - -enum class ScrollDirection { UP, DOWN, LEFT, RIGHT } diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentResult.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentResult.kt deleted file mode 100644 index 03560f1..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentResult.kt +++ /dev/null @@ -1,20 +0,0 @@ -package dev.shantoislam.agenticwebview.models - -sealed class AgentResult { - data class Success(val data: T) : AgentResult() - data class Error(val error: AgentError) : AgentResult() -} - -sealed class AgentError { - data class JsEvaluationTimeout(val timeoutMs: Long) : AgentError() - data class JsEvaluationFailed(val message: String) : AgentError() - data class ElementNotFound(val agentId: String) : AgentError() - data class ElementOccluded(val agentId: String, val occludedBy: String?) : AgentError() - data class NavigationFailed(val url: String, val httpCode: Int?) : AgentError() - data class WebViewCrashed(val didRecover: Boolean) : AgentError() - data class ScreenshotFailed(val reason: String) : AgentError() - data class PageNotReady(val currentState: PageLifecycleState) : AgentError() - data class Timeout(val operation: String, val timeoutMs: Long) : AgentError() - data class FileUploaderDetected(val agentId: String) : AgentError() - data class NoNavigationHistory(val direction: String) : AgentError() -} diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentState.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentState.kt deleted file mode 100644 index 32002f5..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/AgentState.kt +++ /dev/null @@ -1,27 +0,0 @@ -package dev.shantoislam.agenticwebview.models - -import kotlinx.serialization.Serializable - -@Serializable -data class AgentState( - val accessibilityTree: String, - val screenshotBase64: String?, - val viewportInfo: ViewportInfo, - val url: String, - val title: String, - val pageState: PageLifecycleState, - val elementCount: Int, - val truncated: Boolean, - val selectorMap: Map? = null, - val compactTree: String? = null -) - -@Serializable -data class ViewportInfo( - val devicePixelRatio: Double, - val visualViewportScale: Double, - val scrollX: Int, - val scrollY: Int, - val viewportWidth: Int, - val viewportHeight: Int -) diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/DropdownOption.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/DropdownOption.kt deleted file mode 100644 index f68b413..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/DropdownOption.kt +++ /dev/null @@ -1,10 +0,0 @@ -package dev.shantoislam.agenticwebview.models - -import kotlinx.serialization.Serializable - -@Serializable -data class DropdownOption( - val value: String, - val text: String, - val index: Int -) diff --git a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/PageLifecycleState.kt b/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/PageLifecycleState.kt deleted file mode 100644 index f703fd9..0000000 --- a/agentic-webview/src/main/java/dev/shantoislam/agenticwebview/models/PageLifecycleState.kt +++ /dev/null @@ -1,13 +0,0 @@ -package dev.shantoislam.agenticwebview.models - -import kotlinx.serialization.Serializable - -@Serializable -enum class PageLifecycleState { - IDLE, - LOADING, - INTERACTIVE, - COMPLETE, - ERROR, - CRASHED -} diff --git a/agentic-webview/src/main/res/drawable/ic_agentic_logo.xml b/agentic-webview/src/main/res/drawable/ic_agentic_logo.xml deleted file mode 100644 index ec435d0..0000000 --- a/agentic-webview/src/main/res/drawable/ic_agentic_logo.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/agentic-webview/src/test/java/dev/shantoislam/agenticwebview/internal/JsUtilsTest.kt b/agentic-webview/src/test/java/dev/shantoislam/agenticwebview/internal/JsUtilsTest.kt deleted file mode 100644 index e8531ff..0000000 --- a/agentic-webview/src/test/java/dev/shantoislam/agenticwebview/internal/JsUtilsTest.kt +++ /dev/null @@ -1,22 +0,0 @@ -package dev.shantoislam.agenticwebview.internal - -import dev.shantoislam.agenticwebview.internal.JsUtils -import org.junit.Assert.assertEquals -import org.junit.Test - -class JsUtilsTest { - - @Test - fun testEscapeJs() { - val input = "Hello 'World' \"Quotes\" \\ Backslash \n Newline" - val expected = "Hello \\'World\\' \\\"Quotes\\\" \\\\ Backslash \\n Newline" - assertEquals(expected, JsUtils.escapeJs(input)) - } - - @Test - fun testEscapeJsInjection() { - val input = "'); alert('XSS'); //" - val expected = "\\'); alert(\\'XSS\\'); //" - assertEquals(expected, JsUtils.escapeJs(input)) - } -} diff --git a/app/src/androidTest/java/dev/shantoislam/agenticwebview/app/ExampleInstrumentedTest.kt b/app/src/androidTest/java/dev/shantoislam/agenticwebview/app/ExampleInstrumentedTest.kt deleted file mode 100644 index 1ac512a..0000000 --- a/app/src/androidTest/java/dev/shantoislam/agenticwebview/app/ExampleInstrumentedTest.kt +++ /dev/null @@ -1,24 +0,0 @@ -package dev.shantoislam.agenticwebview.app - -import androidx.test.platform.app.InstrumentationRegistry -import androidx.test.ext.junit.runners.AndroidJUnit4 - -import org.junit.Test -import org.junit.runner.RunWith - -import org.junit.Assert.* - -/** - * Instrumented test, which will execute on an Android device. - * - * See [testing documentation](http://d.android.com/tools/testing). - */ -@RunWith(AndroidJUnit4::class) -class ExampleInstrumentedTest { - @Test - fun useAppContext() { - // Context of the app under test. - val appContext = InstrumentationRegistry.getInstrumentation().targetContext - assertEquals("dev.shantoislam.agenticwebview.app", appContext.packageName) - } -} \ No newline at end of file diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/MainActivity.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/MainActivity.kt deleted file mode 100644 index a141404..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/MainActivity.kt +++ /dev/null @@ -1,508 +0,0 @@ -package dev.shantoislam.agenticwebview.app - -import android.os.Bundle -import androidx.activity.ComponentActivity -import androidx.activity.compose.setContent -import androidx.activity.enableEdgeToEdge -import androidx.activity.viewModels -import androidx.compose.animation.* -import androidx.compose.animation.core.* -import androidx.compose.foundation.background -import androidx.compose.foundation.layout.* -import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.items -import androidx.compose.foundation.shape.CircleShape -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.automirrored.filled.ArrowBack -import androidx.compose.material.icons.automirrored.filled.ArrowForward -import androidx.compose.material.icons.automirrored.filled.Send -import androidx.compose.material.icons.filled.* -import androidx.compose.material3.* -import androidx.compose.runtime.* -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.clip -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.vector.ImageVector -import androidx.compose.ui.text.font.FontWeight -import androidx.compose.ui.text.input.ImeAction -import androidx.compose.ui.tooling.preview.Preview -import androidx.compose.ui.unit.dp -import androidx.compose.ui.unit.sp -import kotlinx.coroutines.launch -import androidx.compose.foundation.text.KeyboardActions -import androidx.compose.foundation.text.KeyboardOptions -import dev.shantoislam.agenticwebview.models.AgentAction -import dev.shantoislam.agenticwebview.AgenticWebView -import dev.shantoislam.agenticwebview.AgenticWebViewComposable -import dev.shantoislam.agenticwebview.app.ui.AgenticWebViewModel -import dev.shantoislam.agenticwebview.app.ui.ChatMessage -import dev.shantoislam.agenticwebview.app.ui.AgentSettings -import dev.shantoislam.agenticwebview.app.ui.theme.AppTheme - -class MainActivity : ComponentActivity() { - private val viewModel: AgenticWebViewModel by viewModels() - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - AgenticWebView.init() - enableEdgeToEdge() - setContent { - AppTheme { - MainScreen(viewModel) - } - } - } -} - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -fun MainScreen(viewModel: AgenticWebViewModel) { - var showSettings by remember { mutableStateOf(false) } - val messages = viewModel.messages - val isThinking by viewModel.isThinking.collectAsState() - val loadingProgress by viewModel.loadingProgress.collectAsState() - val settings by viewModel.settings.collectAsState() - val scope = rememberCoroutineScope() - val scaffoldState = rememberBottomSheetScaffoldState() - var canGoBack by remember { mutableStateOf(false) } - var canGoForward by remember { mutableStateOf(false) } - - LaunchedEffect(Unit) { - canGoBack = viewModel.controller.canGoBack() - canGoForward = viewModel.controller.canGoForward() - } - - fun dispatchAction(action: AgentAction) { - scope.launch { - viewModel.controller.executeAction(action) - canGoBack = viewModel.controller.canGoBack() - canGoForward = viewModel.controller.canGoForward() - } - } - - BottomSheetScaffold( - scaffoldState = scaffoldState, - sheetPeekHeight = 120.dp, - sheetShape = RoundedCornerShape(topStart = 28.dp, topEnd = 28.dp), - sheetDragHandle = { BottomSheetDefaults.DragHandle() }, - topBar = { - BrowserTopBar( - progress = loadingProgress, - onNavigate = { url -> dispatchAction(AgentAction.Navigate(url)) }, - onSettingsClick = { showSettings = true }, - onGoBack = { dispatchAction(AgentAction.GoBack) }, - onGoForward = { dispatchAction(AgentAction.GoForward) }, - onRefresh = { dispatchAction(AgentAction.Refresh) }, - canGoBack = canGoBack, - canGoForward = canGoForward - ) - }, - sheetContent = { - AgentChatArea( - messages = messages, - isThinking = isThinking, - onSend = { viewModel.sendMessage(it) } - ) - } - ) { innerPadding -> - Box( - modifier = Modifier - .padding(innerPadding) - .fillMaxSize() - .background(MaterialTheme.colorScheme.surface) - ) { - AgenticWebViewComposable( - controller = viewModel.controller, - modifier = Modifier.fillMaxSize() - ) - } - - if (showSettings) { - SettingsSheet( - settings = settings, - onDismiss = { showSettings = false }, - onSave = { viewModel.updateSettings(it) } - ) - } - } -} - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -fun BrowserTopBar( - progress: Int, - onNavigate: (String) -> Unit, - onSettingsClick: () -> Unit, - onGoBack: () -> Unit, - onGoForward: () -> Unit, - onRefresh: () -> Unit, - canGoBack: Boolean, - canGoForward: Boolean -) { - var urlText by remember { mutableStateOf("https://www.google.com") } - - Surface( - color = MaterialTheme.colorScheme.surfaceContainer, - tonalElevation = 2.dp - ) { - Column { - CenterAlignedTopAppBar( - navigationIcon = { - Row { - IconButton(onClick = onGoBack, enabled = canGoBack) { - Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Back") - } - IconButton(onClick = onGoForward, enabled = canGoForward) { - Icon(Icons.AutoMirrored.Filled.ArrowForward, contentDescription = "Forward") - } - IconButton(onClick = onRefresh) { - Icon(Icons.Default.Refresh, contentDescription = "Refresh") - } - } - }, - title = { - TextField( - value = urlText, - onValueChange = { urlText = it }, - modifier = Modifier - .fillMaxWidth(0.9f) - .height(48.dp), - colors = TextFieldDefaults.colors( - focusedIndicatorColor = Color.Transparent, - unfocusedIndicatorColor = Color.Transparent, - focusedContainerColor = MaterialTheme.colorScheme.surfaceContainerHighest, - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceContainerHighest - ), - leadingIcon = { Icon(Icons.Default.Search, contentDescription = null, modifier = Modifier.size(20.dp)) }, - trailingIcon = { - if (urlText.isNotEmpty()) { - IconButton(onClick = { urlText = "" }) { - Icon(Icons.Default.Close, contentDescription = "Clear", modifier = Modifier.size(18.dp)) - } - } - }, - singleLine = true, - shape = CircleShape, - textStyle = MaterialTheme.typography.bodyMedium, - keyboardOptions = KeyboardOptions(imeAction = ImeAction.Go), - keyboardActions = KeyboardActions(onGo = { onNavigate(urlText) }) - ) - }, - actions = { - IconButton(onClick = onSettingsClick) { - Icon(Icons.Default.Settings, contentDescription = "Settings") - } - }, - colors = TopAppBarDefaults.topAppBarColors( - containerColor = Color.Transparent - ) - ) - - if (progress < 100) { - LinearProgressIndicator( - progress = { progress / 100f }, - modifier = Modifier.fillMaxWidth().height(2.dp), - color = MaterialTheme.colorScheme.primary, - trackColor = Color.Transparent - ) - } else { - Spacer(modifier = Modifier.height(2.dp)) - } - } - } -} - -@Composable -fun AgentChatArea( - messages: List, - isThinking: Boolean, - onSend: (String) -> Unit -) { - Column( - modifier = Modifier - .fillMaxWidth() - .heightIn(min = 400.dp, max = 600.dp) - .padding(bottom = 16.dp) - ) { - Row( - modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 24.dp, vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically - ) { - Text( - "Agentic Assistant", - style = MaterialTheme.typography.titleMedium, - fontWeight = FontWeight.Bold, - modifier = Modifier.weight(1f) - ) - - ThinkingIndicator(isThinking) - } - - LazyColumn( - modifier = Modifier - .weight(1f) - .fillMaxWidth() - .padding(horizontal = 16.dp), - contentPadding = PaddingValues(vertical = 8.dp) - ) { - items(messages) { message -> - ChatBubble(message) - } - } - - ChatInput(onSend = onSend) - } -} - -@Composable -fun ThinkingIndicator(isThinking: Boolean) { - Row(verticalAlignment = Alignment.CenterVertically) { - val infiniteTransition = rememberInfiniteTransition(label = "thinking") - val alpha by infiniteTransition.animateFloat( - initialValue = 0.3f, - targetValue = 1f, - animationSpec = infiniteRepeatable( - animation = tween(1000, easing = LinearEasing), - repeatMode = RepeatMode.Reverse - ), - label = "alpha" - ) - - Box( - modifier = Modifier - .size(8.dp) - .clip(CircleShape) - .background(if (isThinking) MaterialTheme.colorScheme.primary.copy(alpha = alpha) else MaterialTheme.colorScheme.outlineVariant) - ) - Spacer(modifier = Modifier.width(8.dp)) - Text( - text = if (isThinking) "Thinking..." else "Ready", - style = MaterialTheme.typography.labelMedium, - color = if (isThinking) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant - ) - } -} - -@Composable -fun ChatBubble(message: ChatMessage) { - val bubbleColor = if (message.isUser) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.secondaryContainer - val contentColor = if (message.isUser) MaterialTheme.colorScheme.onPrimary else MaterialTheme.colorScheme.onSecondaryContainer - - val shape = if (message.isUser) { - RoundedCornerShape(20.dp, 4.dp, 20.dp, 20.dp) - } else { - RoundedCornerShape(4.dp, 20.dp, 20.dp, 20.dp) - } - - Column( - modifier = Modifier - .fillMaxWidth() - .padding(vertical = 4.dp), - horizontalAlignment = if (message.isUser) Alignment.End else Alignment.Start - ) { - Surface( - color = bubbleColor, - contentColor = contentColor, - shape = shape, - tonalElevation = 1.dp - ) { - Text( - text = message.message, - modifier = Modifier.padding(16.dp, 10.dp), - style = MaterialTheme.typography.bodyMedium.copy(lineHeight = 20.sp) - ) - } - } -} - -@Composable -fun ChatInput(onSend: (String) -> Unit) { - var text by remember { mutableStateOf("") } - - Surface( - modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 16.dp, vertical = 8.dp), - color = MaterialTheme.colorScheme.surface, - shape = CircleShape, - tonalElevation = 3.dp, - shadowElevation = 2.dp - ) { - Row( - modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 4.dp, vertical = 4.dp), - verticalAlignment = Alignment.CenterVertically - ) { - TextField( - value = text, - onValueChange = { text = it }, - modifier = Modifier - .weight(1f) - .heightIn(min = 48.dp), - placeholder = { Text("Ask your agent...", style = MaterialTheme.typography.bodyMedium) }, - colors = TextFieldDefaults.colors( - focusedIndicatorColor = Color.Transparent, - unfocusedIndicatorColor = Color.Transparent, - focusedContainerColor = Color.Transparent, - unfocusedContainerColor = Color.Transparent - ), - singleLine = false, - maxLines = 4 - ) - - IconButton( - onClick = { - if (text.isNotBlank()) { - onSend(text) - text = "" - } - }, - enabled = text.isNotBlank(), - colors = IconButtonDefaults.filledIconButtonColors( - containerColor = MaterialTheme.colorScheme.primary, - contentColor = MaterialTheme.colorScheme.onPrimary, - disabledContainerColor = MaterialTheme.colorScheme.surfaceVariant, - disabledContentColor = MaterialTheme.colorScheme.onSurfaceVariant - ), - modifier = Modifier.size(40.dp) - ) { - Icon(Icons.AutoMirrored.Filled.Send, contentDescription = "Send", modifier = Modifier.size(20.dp)) - } - } - } -} - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -fun SettingsSheet( - settings: AgentSettings, - onDismiss: () -> Unit, - onSave: (AgentSettings) -> Unit -) { - var localSettings by remember { mutableStateOf(settings) } - - ModalBottomSheet( - onDismissRequest = onDismiss, - sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) - ) { - Column( - modifier = Modifier - .padding(24.dp) - .fillMaxWidth() - .navigationBarsPadding() - ) { - Text( - "Agent Configuration", - style = MaterialTheme.typography.headlineSmall, - fontWeight = FontWeight.Bold - ) - Spacer(modifier = Modifier.height(24.dp)) - - Surface( - color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.3f), - shape = RoundedCornerShape(16.dp) - ) { - Row( - modifier = Modifier.padding(16.dp), - verticalAlignment = Alignment.CenterVertically - ) { - Column(modifier = Modifier.weight(1f)) { - Text("Simulation Mode", style = MaterialTheme.typography.titleMedium) - Text( - "Use mock responses for testing", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant - ) - } - Switch( - checked = localSettings.useSimulation, - onCheckedChange = { localSettings = localSettings.copy(useSimulation = it) } - ) - } - } - - AnimatedVisibility( - visible = !localSettings.useSimulation, - enter = expandVertically() + fadeIn(), - exit = shrinkVertically() + fadeOut() - ) { - Column { - Spacer(modifier = Modifier.height(24.dp)) - OutlinedTextField( - value = localSettings.baseUrl, - onValueChange = { localSettings = localSettings.copy(baseUrl = it) }, - label = { Text("Base URL") }, - modifier = Modifier.fillMaxWidth(), - shape = RoundedCornerShape(12.dp) - ) - Spacer(modifier = Modifier.height(12.dp)) - OutlinedTextField( - value = localSettings.apiKey, - onValueChange = { localSettings = localSettings.copy(apiKey = it) }, - label = { Text("API Key") }, - modifier = Modifier.fillMaxWidth(), - shape = RoundedCornerShape(12.dp) - ) - Spacer(modifier = Modifier.height(12.dp)) - OutlinedTextField( - value = localSettings.modelName, - onValueChange = { localSettings = localSettings.copy(modelName = it) }, - label = { Text("Model Name") }, - modifier = Modifier.fillMaxWidth(), - shape = RoundedCornerShape(12.dp) - ) - } - } - - Spacer(modifier = Modifier.height(32.dp)) - Button( - onClick = { - onSave(localSettings) - onDismiss() - }, - modifier = Modifier.fillMaxWidth().height(56.dp), - shape = RoundedCornerShape(16.dp) - ) { - Text("Save Settings", style = MaterialTheme.typography.titleMedium) - } - Spacer(modifier = Modifier.height(16.dp)) - } - } -} - -@Preview(showBackground = true) -@Composable -fun BrowserTopBarPreview() { - AppTheme { - BrowserTopBar(progress = 45, onNavigate = {}, onSettingsClick = {}, onGoBack = {}, onGoForward = {}, onRefresh = {}, canGoBack = true, canGoForward = false) - } -} - -@Preview(showBackground = true) -@Composable -fun ChatBubbleUserPreview() { - AppTheme { - ChatBubble(ChatMessage("User", "Hello agent, can you help me?", true)) - } -} - -@Preview(showBackground = true) -@Composable -fun ChatBubbleAgentPreview() { - AppTheme { - ChatBubble(ChatMessage("Agent", "Sure! I can help you with that. What do you need?", false)) - } -} - -@Preview(showBackground = true) -@Composable -fun ChatInputPreview() { - AppTheme { - ChatInput(onSend = {}) - } -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/agent/AgenticWebviewTools.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/agent/AgenticWebviewTools.kt deleted file mode 100644 index c48740e..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/agent/AgenticWebviewTools.kt +++ /dev/null @@ -1,91 +0,0 @@ -package dev.shantoislam.agenticwebview.app.agent - -import ai.koog.agents.core.tools.reflect.ToolSet -import ai.koog.agents.core.tools.annotations.Tool -import ai.koog.agents.core.tools.annotations.LLMDescription -import dev.shantoislam.agenticwebview.AgenticWebController -import dev.shantoislam.agenticwebview.models.AgentAction -import dev.shantoislam.agenticwebview.models.AgentResult -import dev.shantoislam.agenticwebview.models.ScrollDirection -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json - -class AgenticWebviewTools(private val controller: AgenticWebController) : ToolSet { - - @Tool - @LLMDescription("Captures the current state of the webview including the accessibility tree, viewport info, and URL.") - suspend fun webview_get_state(): String { - return when (val result = controller.captureState()) { - is AgentResult.Success -> Json.encodeToString(result.data) - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Navigates the webview to the specified URL.") - suspend fun webview_navigate(url: String): String { - return when (val result = controller.executeAction(AgentAction.Navigate(url))) { - is AgentResult.Success -> "Navigated to $url" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Clicks on an element with the given agent ID.") - suspend fun webview_click(agentId: String): String { - return when (val result = controller.executeAction(AgentAction.Click(agentId))) { - is AgentResult.Success -> "Clicked element $agentId" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Inputs text into an element with the given agent ID.") - suspend fun webview_input_text(agentId: String, text: String): String { - return when (val result = controller.executeAction(AgentAction.InputText(agentId, text))) { - is AgentResult.Success -> "Typed text into $agentId" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Scrolls the webview in a specified direction (UP, DOWN, LEFT, RIGHT) by a ratio (0.0 to 1.0).") - suspend fun webview_scroll(direction: String, ratio: Float = 0.5f): String { - val scrollDirection = try { - ScrollDirection.valueOf(direction.uppercase()) - } catch (e: Exception) { - return "Error: Invalid direction $direction" - } - return when (val result = controller.executeAction(AgentAction.Scroll(scrollDirection, ratio))) { - is AgentResult.Success -> "Scrolled ${scrollDirection.name}" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Navigates back in the browser history.") - suspend fun webview_go_back(): String { - return when (val result = controller.executeAction(AgentAction.GoBack)) { - is AgentResult.Success -> "Navigated back" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Navigates forward in the browser history.") - suspend fun webview_go_forward(): String { - return when (val result = controller.executeAction(AgentAction.GoForward)) { - is AgentResult.Success -> "Navigated forward" - is AgentResult.Error -> "Error: ${result.error}" - } - } - - @Tool - @LLMDescription("Refreshes the current page.") - suspend fun webview_refresh(): String { - return when (val result = controller.executeAction(AgentAction.Refresh)) { - is AgentResult.Success -> "Page refreshed" - is AgentResult.Error -> "Error: ${result.error}" - } - } -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsDao.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsDao.kt deleted file mode 100644 index d05e0e2..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsDao.kt +++ /dev/null @@ -1,16 +0,0 @@ -package dev.shantoislam.agenticwebview.app.model - -import androidx.room.Dao -import androidx.room.Insert -import androidx.room.OnConflictStrategy -import androidx.room.Query -import kotlinx.coroutines.flow.Flow - -@Dao -interface AgentSettingsDao { - @Query("SELECT * FROM agent_settings WHERE id = 0") - fun getSettings(): Flow - - @Insert(onConflict = OnConflictStrategy.REPLACE) - suspend fun insertSettings(settings: AgentSettingsEntity) -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsEntity.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsEntity.kt deleted file mode 100644 index d4448c8..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AgentSettingsEntity.kt +++ /dev/null @@ -1,13 +0,0 @@ -package dev.shantoislam.agenticwebview.app.model - -import androidx.room.Entity -import androidx.room.PrimaryKey - -@Entity(tableName = "agent_settings") -data class AgentSettingsEntity( - @PrimaryKey val id: Int = 0, // Single row for settings - val useSimulation: Boolean, - val baseUrl: String, - val apiKey: String, - val modelName: String -) diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AppDatabase.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AppDatabase.kt deleted file mode 100644 index b430425..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/model/AppDatabase.kt +++ /dev/null @@ -1,28 +0,0 @@ -package dev.shantoislam.agenticwebview.app.model - -import android.content.Context -import androidx.room.Database -import androidx.room.Room -import androidx.room.RoomDatabase - -@Database(entities = [AgentSettingsEntity::class], version = 1) -abstract class AppDatabase : RoomDatabase() { - abstract fun agentSettingsDao(): AgentSettingsDao - - companion object { - @Volatile - private var INSTANCE: AppDatabase? = null - - fun getDatabase(context: Context): AppDatabase { - return INSTANCE ?: synchronized(this) { - val instance = Room.databaseBuilder( - context.applicationContext, - AppDatabase::class.java, - "agentic_webview_db" - ).build() - INSTANCE = instance - instance - } - } - } -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/AgenticWebViewModel.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/AgenticWebViewModel.kt deleted file mode 100644 index 7d3b316..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/AgenticWebViewModel.kt +++ /dev/null @@ -1,181 +0,0 @@ -package dev.shantoislam.agenticwebview.app.ui - -import androidx.compose.runtime.mutableStateListOf -import androidx.lifecycle.viewModelScope -import dev.shantoislam.agenticwebview.AgenticWebController -import dev.shantoislam.agenticwebview.config.AgenticWebViewConfig -import dev.shantoislam.agenticwebview.app.agent.* -import dev.shantoislam.agenticwebview.app.model.AgentSettingsEntity -import dev.shantoislam.agenticwebview.app.model.AppDatabase -import ai.koog.agents.core.agent.AIAgent -import ai.koog.agents.core.agent.singleRunStrategy -import ai.koog.agents.core.tools.ToolRegistryBuilder -import ai.koog.agents.core.tools.reflect.asTools -import ai.koog.prompt.executor.clients.openai.OpenAILLMClient -import ai.koog.prompt.executor.clients.openai.OpenAIClientSettings -import ai.koog.prompt.executor.llms.MultiLLMPromptExecutor -import ai.koog.prompt.llm.LLModel -import ai.koog.prompt.llm.LLMProvider -import ai.koog.prompt.llm.LLMCapability -import android.app.Application -import androidx.lifecycle.AndroidViewModel -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.collectLatest -import kotlinx.coroutines.launch -import kotlinx.coroutines.delay - -data class ChatMessage( - val sender: String, - val message: String, - val isUser: Boolean -) - -data class AgentSettings( - val useSimulation: Boolean = true, - val baseUrl: String = "https://api.openai.com", - val apiKey: String = "", - val modelName: String = "gpt-4-turbo" -) - -fun AgentSettings.toEntity() = AgentSettingsEntity( - useSimulation = useSimulation, - baseUrl = baseUrl, - apiKey = apiKey, - modelName = modelName -) - -fun AgentSettingsEntity.toDomain() = AgentSettings( - useSimulation = useSimulation, - baseUrl = baseUrl, - apiKey = apiKey, - modelName = modelName -) - -class AgenticWebViewModel(application: Application) : AndroidViewModel(application) { - private val db = AppDatabase.getDatabase(application) - private val dao = db.agentSettingsDao() - - val controller = AgenticWebController(AgenticWebViewConfig()) - - private val _messages = mutableStateListOf() - val messages: List = _messages - - private val _settings = MutableStateFlow(AgentSettings()) - val settings: StateFlow = _settings.asStateFlow() - - private val _isThinking = MutableStateFlow(false) - val isThinking: StateFlow = _isThinking.asStateFlow() - - val loadingProgress: StateFlow = controller.loadingProgress - - init { - _messages.add(ChatMessage("Agent", "Hello! I'm your Agentic WebView assistant. How can I help you today?", false)) - - viewModelScope.launch { - dao.getSettings().collectLatest { entity -> - entity?.let { - _settings.value = it.toDomain() - } - } - } - } - - fun updateSettings(newSettings: AgentSettings) { - viewModelScope.launch { - dao.insertSettings(newSettings.toEntity()) - } - } - - fun sendMessage(text: String) { - if (text.isBlank()) return - - _messages.add(ChatMessage("User", text, true)) - - viewModelScope.launch { - _isThinking.value = true - try { - if (_settings.value.useSimulation) { - runSimulation(text) - } else { - runLiveAgent(text) - } - } catch (e: Exception) { - _messages.add(ChatMessage("Agent", "Error: ${e.message}", false)) - } finally { - _isThinking.value = false - } - } - } - - private suspend fun runSimulation(text: String) { - delay(1000) - _messages.add(ChatMessage("Agent", "Thinking about: $text", false)) - - if (text.contains("google", ignoreCase = true)) { - _messages.add(ChatMessage("Agent", "Navigating to Google...", false)) - controller.executeAction(dev.shantoislam.agenticwebview.models.AgentAction.Navigate("https://www.google.com")) - } else if (text.contains("search", ignoreCase = true)) { - _messages.add(ChatMessage("Agent", "Searching...", false)) - } else { - _messages.add(ChatMessage("Agent", "I'm in simulation mode. Try asking to go to google.", false)) - } - } - - private suspend fun runLiveAgent(text: String) { - val currentSettings = _settings.value - if (currentSettings.apiKey.isBlank()) { - _messages.add(ChatMessage("Agent", "Please set an API Key in settings.", false)) - return - } - - _messages.add(ChatMessage("Agent", "Connecting to live agent (Koog)...", false)) - - val tools = AgenticWebviewTools(controller) - val registry = ToolRegistryBuilder().apply { - tools(AgenticWebviewTools::class.asTools(tools)) - }.build() - - // Configure custom OpenAI client for custom base URL support - val clientSettings = OpenAIClientSettings( - baseUrl = currentSettings.baseUrl - ) - val llmClient = OpenAILLMClient(currentSettings.apiKey, clientSettings) - val promptExecutor = MultiLLMPromptExecutor(llmClient) - - // Define custom model with hardcoded constraints: 128k context, 64k max output - val customModel = LLModel( - provider = LLMProvider.OpenAI, - id = currentSettings.modelName, - capabilities = listOf( - LLMCapability.Temperature, - LLMCapability.Tools, - LLMCapability.Completion, - LLMCapability.OpenAIEndpoint.Completions, - LLMCapability.Vision.Image - ), - contextLength = 128000L, - maxOutputTokens = 64000L - ) - - val agent = AIAgent( - promptExecutor = promptExecutor, - llmModel = customModel, - strategy = singleRunStrategy(), - systemPrompt = """ - You are a web browsing agent. Use the provided tools to interact with the webview. - Always start by calling 'webview_get_state' to see what's on the page. - When you are done or have found the answer, speak to the user. - """.trimIndent(), - toolRegistry = registry - ) - - try { - val response = agent.run(text) - _messages.add(ChatMessage("Agent", response.toString(), false)) - } catch (e: Exception) { - _messages.add(ChatMessage("Agent", "Koog Error: ${e.message}", false)) - } - } -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Color.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Color.kt deleted file mode 100644 index 35e25d2..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Color.kt +++ /dev/null @@ -1,219 +0,0 @@ -package dev.shantoislam.agenticwebview.app.ui.theme - -import androidx.compose.ui.graphics.Color - -val primaryLight = Color(0xFF415F91) -val onPrimaryLight = Color(0xFFFFFFFF) -val primaryContainerLight = Color(0xFFD6E3FF) -val onPrimaryContainerLight = Color(0xFF284777) -val secondaryLight = Color(0xFF565F71) -val onSecondaryLight = Color(0xFFFFFFFF) -val secondaryContainerLight = Color(0xFFDAE2F9) -val onSecondaryContainerLight = Color(0xFF3E4759) -val tertiaryLight = Color(0xFF705575) -val onTertiaryLight = Color(0xFFFFFFFF) -val tertiaryContainerLight = Color(0xFFFAD8FD) -val onTertiaryContainerLight = Color(0xFF573E5C) -val errorLight = Color(0xFFBA1A1A) -val onErrorLight = Color(0xFFFFFFFF) -val errorContainerLight = Color(0xFFFFDAD6) -val onErrorContainerLight = Color(0xFF93000A) -val backgroundLight = Color(0xFFF9F9FF) -val onBackgroundLight = Color(0xFF191C20) -val surfaceLight = Color(0xFFF9F9FF) -val onSurfaceLight = Color(0xFF191C20) -val surfaceVariantLight = Color(0xFFE0E2EC) -val onSurfaceVariantLight = Color(0xFF44474E) -val outlineLight = Color(0xFF74777F) -val outlineVariantLight = Color(0xFFC4C6D0) -val scrimLight = Color(0xFF000000) -val inverseSurfaceLight = Color(0xFF2E3036) -val inverseOnSurfaceLight = Color(0xFFF0F0F7) -val inversePrimaryLight = Color(0xFFAAC7FF) -val surfaceDimLight = Color(0xFFD9D9E0) -val surfaceBrightLight = Color(0xFFF9F9FF) -val surfaceContainerLowestLight = Color(0xFFFFFFFF) -val surfaceContainerLowLight = Color(0xFFF3F3FA) -val surfaceContainerLight = Color(0xFFEDEDF4) -val surfaceContainerHighLight = Color(0xFFE7E8EE) -val surfaceContainerHighestLight = Color(0xFFE2E2E9) - -val primaryLightMediumContrast = Color(0xFF133665) -val onPrimaryLightMediumContrast = Color(0xFFFFFFFF) -val primaryContainerLightMediumContrast = Color(0xFF506DA0) -val onPrimaryContainerLightMediumContrast = Color(0xFFFFFFFF) -val secondaryLightMediumContrast = Color(0xFF2E3647) -val onSecondaryLightMediumContrast = Color(0xFFFFFFFF) -val secondaryContainerLightMediumContrast = Color(0xFF646D80) -val onSecondaryContainerLightMediumContrast = Color(0xFFFFFFFF) -val tertiaryLightMediumContrast = Color(0xFF452E4A) -val onTertiaryLightMediumContrast = Color(0xFFFFFFFF) -val tertiaryContainerLightMediumContrast = Color(0xFF7F6484) -val onTertiaryContainerLightMediumContrast = Color(0xFFFFFFFF) -val errorLightMediumContrast = Color(0xFF740006) -val onErrorLightMediumContrast = Color(0xFFFFFFFF) -val errorContainerLightMediumContrast = Color(0xFFCF2C27) -val onErrorContainerLightMediumContrast = Color(0xFFFFFFFF) -val backgroundLightMediumContrast = Color(0xFFF9F9FF) -val onBackgroundLightMediumContrast = Color(0xFF191C20) -val surfaceLightMediumContrast = Color(0xFFF9F9FF) -val onSurfaceLightMediumContrast = Color(0xFF0F1116) -val surfaceVariantLightMediumContrast = Color(0xFFE0E2EC) -val onSurfaceVariantLightMediumContrast = Color(0xFF33363E) -val outlineLightMediumContrast = Color(0xFF4F525A) -val outlineVariantLightMediumContrast = Color(0xFF6A6D75) -val scrimLightMediumContrast = Color(0xFF000000) -val inverseSurfaceLightMediumContrast = Color(0xFF2E3036) -val inverseOnSurfaceLightMediumContrast = Color(0xFFF0F0F7) -val inversePrimaryLightMediumContrast = Color(0xFFAAC7FF) -val surfaceDimLightMediumContrast = Color(0xFFC5C6CD) -val surfaceBrightLightMediumContrast = Color(0xFFF9F9FF) -val surfaceContainerLowestLightMediumContrast = Color(0xFFFFFFFF) -val surfaceContainerLowLightMediumContrast = Color(0xFFF3F3FA) -val surfaceContainerLightMediumContrast = Color(0xFFE7E8EE) -val surfaceContainerHighLightMediumContrast = Color(0xFFDCDCE3) -val surfaceContainerHighestLightMediumContrast = Color(0xFFD1D1D8) - -val primaryLightHighContrast = Color(0xFF032B5B) -val onPrimaryLightHighContrast = Color(0xFFFFFFFF) -val primaryContainerLightHighContrast = Color(0xFF2A497A) -val onPrimaryContainerLightHighContrast = Color(0xFFFFFFFF) -val secondaryLightHighContrast = Color(0xFF232C3D) -val onSecondaryLightHighContrast = Color(0xFFFFFFFF) -val secondaryContainerLightHighContrast = Color(0xFF41495B) -val onSecondaryContainerLightHighContrast = Color(0xFFFFFFFF) -val tertiaryLightHighContrast = Color(0xFF3A2440) -val onTertiaryLightHighContrast = Color(0xFFFFFFFF) -val tertiaryContainerLightHighContrast = Color(0xFF59405E) -val onTertiaryContainerLightHighContrast = Color(0xFFFFFFFF) -val errorLightHighContrast = Color(0xFF600004) -val onErrorLightHighContrast = Color(0xFFFFFFFF) -val errorContainerLightHighContrast = Color(0xFF98000A) -val onErrorContainerLightHighContrast = Color(0xFFFFFFFF) -val backgroundLightHighContrast = Color(0xFFF9F9FF) -val onBackgroundLightHighContrast = Color(0xFF191C20) -val surfaceLightHighContrast = Color(0xFFF9F9FF) -val onSurfaceLightHighContrast = Color(0xFF000000) -val surfaceVariantLightHighContrast = Color(0xFFE0E2EC) -val onSurfaceVariantLightHighContrast = Color(0xFF000000) -val outlineLightHighContrast = Color(0xFF292C33) -val outlineVariantLightHighContrast = Color(0xFF464951) -val scrimLightHighContrast = Color(0xFF000000) -val inverseSurfaceLightHighContrast = Color(0xFF2E3036) -val inverseOnSurfaceLightHighContrast = Color(0xFFFFFFFF) -val inversePrimaryLightHighContrast = Color(0xFFAAC7FF) -val surfaceDimLightHighContrast = Color(0xFFB8B8BF) -val surfaceBrightLightHighContrast = Color(0xFFF9F9FF) -val surfaceContainerLowestLightHighContrast = Color(0xFFFFFFFF) -val surfaceContainerLowLightHighContrast = Color(0xFFF0F0F7) -val surfaceContainerLightHighContrast = Color(0xFFE2E2E9) -val surfaceContainerHighLightHighContrast = Color(0xFFD3D4DB) -val surfaceContainerHighestLightHighContrast = Color(0xFFC5C6CD) - -val primaryDark = Color(0xFFAAC7FF) -val onPrimaryDark = Color(0xFF0A305F) -val primaryContainerDark = Color(0xFF284777) -val onPrimaryContainerDark = Color(0xFFD6E3FF) -val secondaryDark = Color(0xFFBEC6DC) -val onSecondaryDark = Color(0xFF283141) -val secondaryContainerDark = Color(0xFF3E4759) -val onSecondaryContainerDark = Color(0xFFDAE2F9) -val tertiaryDark = Color(0xFFDDBCE0) -val onTertiaryDark = Color(0xFF3F2844) -val tertiaryContainerDark = Color(0xFF573E5C) -val onTertiaryContainerDark = Color(0xFFFAD8FD) -val errorDark = Color(0xFFFFB4AB) -val onErrorDark = Color(0xFF690005) -val errorContainerDark = Color(0xFF93000A) -val onErrorContainerDark = Color(0xFFFFDAD6) -val backgroundDark = Color(0xFF111318) -val onBackgroundDark = Color(0xFFE2E2E9) -val surfaceDark = Color(0xFF111318) -val onSurfaceDark = Color(0xFFE2E2E9) -val surfaceVariantDark = Color(0xFF44474E) -val onSurfaceVariantDark = Color(0xFFC4C6D0) -val outlineDark = Color(0xFF8E9099) -val outlineVariantDark = Color(0xFF44474E) -val scrimDark = Color(0xFF000000) -val inverseSurfaceDark = Color(0xFFE2E2E9) -val inverseOnSurfaceDark = Color(0xFF2E3036) -val inversePrimaryDark = Color(0xFF415F91) -val surfaceDimDark = Color(0xFF111318) -val surfaceBrightDark = Color(0xFF37393E) -val surfaceContainerLowestDark = Color(0xFF0C0E13) -val surfaceContainerLowDark = Color(0xFF191C20) -val surfaceContainerDark = Color(0xFF1D2024) -val surfaceContainerHighDark = Color(0xFF282A2F) -val surfaceContainerHighestDark = Color(0xFF33353A) - -val primaryDarkMediumContrast = Color(0xFFCDDDFF) -val onPrimaryDarkMediumContrast = Color(0xFF002551) -val primaryContainerDarkMediumContrast = Color(0xFF7491C7) -val onPrimaryContainerDarkMediumContrast = Color(0xFF000000) -val secondaryDarkMediumContrast = Color(0xFFD4DCF2) -val onSecondaryDarkMediumContrast = Color(0xFF1D2636) -val secondaryContainerDarkMediumContrast = Color(0xFF8891A5) -val onSecondaryContainerDarkMediumContrast = Color(0xFF000000) -val tertiaryDarkMediumContrast = Color(0xFFF3D2F7) -val onTertiaryDarkMediumContrast = Color(0xFF331D39) -val tertiaryContainerDarkMediumContrast = Color(0xFFA487A9) -val onTertiaryContainerDarkMediumContrast = Color(0xFF000000) -val errorDarkMediumContrast = Color(0xFFFFD2CC) -val onErrorDarkMediumContrast = Color(0xFF540003) -val errorContainerDarkMediumContrast = Color(0xFFFF5449) -val onErrorContainerDarkMediumContrast = Color(0xFF000000) -val backgroundDarkMediumContrast = Color(0xFF111318) -val onBackgroundDarkMediumContrast = Color(0xFFE2E2E9) -val surfaceDarkMediumContrast = Color(0xFF111318) -val onSurfaceDarkMediumContrast = Color(0xFFFFFFFF) -val surfaceVariantDarkMediumContrast = Color(0xFF44474E) -val onSurfaceVariantDarkMediumContrast = Color(0xFFDADCE6) -val outlineDarkMediumContrast = Color(0xFFAFB2BB) -val outlineVariantDarkMediumContrast = Color(0xFF8E9099) -val scrimDarkMediumContrast = Color(0xFF000000) -val inverseSurfaceDarkMediumContrast = Color(0xFFE2E2E9) -val inverseOnSurfaceDarkMediumContrast = Color(0xFF282A2F) -val inversePrimaryDarkMediumContrast = Color(0xFF294878) -val surfaceDimDarkMediumContrast = Color(0xFF111318) -val surfaceBrightDarkMediumContrast = Color(0xFF43444A) -val surfaceContainerLowestDarkMediumContrast = Color(0xFF06070C) -val surfaceContainerLowDarkMediumContrast = Color(0xFF1B1E22) -val surfaceContainerDarkMediumContrast = Color(0xFF26282D) -val surfaceContainerHighDarkMediumContrast = Color(0xFF313238) -val surfaceContainerHighestDarkMediumContrast = Color(0xFF3C3E43) - -val primaryDarkHighContrast = Color(0xFFEBF0FF) -val onPrimaryDarkHighContrast = Color(0xFF000000) -val primaryContainerDarkHighContrast = Color(0xFFA6C3FC) -val onPrimaryContainerDarkHighContrast = Color(0xFF000B20) -val secondaryDarkHighContrast = Color(0xFFEBF0FF) -val onSecondaryDarkHighContrast = Color(0xFF000000) -val secondaryContainerDarkHighContrast = Color(0xFFBAC3D8) -val onSecondaryContainerDarkHighContrast = Color(0xFF030B1A) -val tertiaryDarkHighContrast = Color(0xFFFFE9FF) -val onTertiaryDarkHighContrast = Color(0xFF000000) -val tertiaryContainerDarkHighContrast = Color(0xFFD8B8DC) -val onTertiaryContainerDarkHighContrast = Color(0xFF16041D) -val errorDarkHighContrast = Color(0xFFFFECE9) -val onErrorDarkHighContrast = Color(0xFF000000) -val errorContainerDarkHighContrast = Color(0xFFFFAEA4) -val onErrorContainerDarkHighContrast = Color(0xFF220001) -val backgroundDarkHighContrast = Color(0xFF111318) -val onBackgroundDarkHighContrast = Color(0xFFE2E2E9) -val surfaceDarkHighContrast = Color(0xFF111318) -val onSurfaceDarkHighContrast = Color(0xFFFFFFFF) -val surfaceVariantDarkHighContrast = Color(0xFF44474E) -val onSurfaceVariantDarkHighContrast = Color(0xFFFFFFFF) -val outlineDarkHighContrast = Color(0xFFEEEFF9) -val outlineVariantDarkHighContrast = Color(0xFFC0C2CC) -val scrimDarkHighContrast = Color(0xFF000000) -val inverseSurfaceDarkHighContrast = Color(0xFFE2E2E9) -val inverseOnSurfaceDarkHighContrast = Color(0xFF000000) -val inversePrimaryDarkHighContrast = Color(0xFF294878) -val surfaceDimDarkHighContrast = Color(0xFF111318) -val surfaceBrightDarkHighContrast = Color(0xFF4E5056) -val surfaceContainerLowestDarkHighContrast = Color(0xFF000000) -val surfaceContainerLowDarkHighContrast = Color(0xFF1D2024) -val surfaceContainerDarkHighContrast = Color(0xFF2E3036) -val surfaceContainerHighDarkHighContrast = Color(0xFF393B41) -val surfaceContainerHighestDarkHighContrast = Color(0xFF45474C) diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Theme.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Theme.kt deleted file mode 100644 index d51c93f..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Theme.kt +++ /dev/null @@ -1,280 +0,0 @@ -package dev.shantoislam.agenticwebview.app.ui.theme - -import android.app.Activity -import android.os.Build -import androidx.compose.foundation.isSystemInDarkTheme -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.lightColorScheme -import androidx.compose.material3.darkColorScheme -import androidx.compose.material3.dynamicDarkColorScheme -import androidx.compose.material3.dynamicLightColorScheme -import androidx.compose.material3.Typography -import androidx.compose.runtime.Composable -import androidx.compose.runtime.Immutable -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.toArgb -import androidx.compose.ui.platform.LocalContext - -private val lightScheme = lightColorScheme( - primary = primaryLight, - onPrimary = onPrimaryLight, - primaryContainer = primaryContainerLight, - onPrimaryContainer = onPrimaryContainerLight, - secondary = secondaryLight, - onSecondary = onSecondaryLight, - secondaryContainer = secondaryContainerLight, - onSecondaryContainer = onSecondaryContainerLight, - tertiary = tertiaryLight, - onTertiary = onTertiaryLight, - tertiaryContainer = tertiaryContainerLight, - onTertiaryContainer = onTertiaryContainerLight, - error = errorLight, - onError = onErrorLight, - errorContainer = errorContainerLight, - onErrorContainer = onErrorContainerLight, - background = backgroundLight, - onBackground = onBackgroundLight, - surface = surfaceLight, - onSurface = onSurfaceLight, - surfaceVariant = surfaceVariantLight, - onSurfaceVariant = onSurfaceVariantLight, - outline = outlineLight, - outlineVariant = outlineVariantLight, - scrim = scrimLight, - inverseSurface = inverseSurfaceLight, - inverseOnSurface = inverseOnSurfaceLight, - inversePrimary = inversePrimaryLight, - surfaceDim = surfaceDimLight, - surfaceBright = surfaceBrightLight, - surfaceContainerLowest = surfaceContainerLowestLight, - surfaceContainerLow = surfaceContainerLowLight, - surfaceContainer = surfaceContainerLight, - surfaceContainerHigh = surfaceContainerHighLight, - surfaceContainerHighest = surfaceContainerHighestLight, -) - -private val darkScheme = darkColorScheme( - primary = primaryDark, - onPrimary = onPrimaryDark, - primaryContainer = primaryContainerDark, - onPrimaryContainer = onPrimaryContainerDark, - secondary = secondaryDark, - onSecondary = onSecondaryDark, - secondaryContainer = secondaryContainerDark, - onSecondaryContainer = onSecondaryContainerDark, - tertiary = tertiaryDark, - onTertiary = onTertiaryDark, - tertiaryContainer = tertiaryContainerDark, - onTertiaryContainer = onTertiaryContainerDark, - error = errorDark, - onError = onErrorDark, - errorContainer = errorContainerDark, - onErrorContainer = onErrorContainerDark, - background = backgroundDark, - onBackground = onBackgroundDark, - surface = surfaceDark, - onSurface = onSurfaceDark, - surfaceVariant = surfaceVariantDark, - onSurfaceVariant = onSurfaceVariantDark, - outline = outlineDark, - outlineVariant = outlineVariantDark, - scrim = scrimDark, - inverseSurface = inverseSurfaceDark, - inverseOnSurface = inverseOnSurfaceDark, - inversePrimary = inversePrimaryDark, - surfaceDim = surfaceDimDark, - surfaceBright = surfaceBrightDark, - surfaceContainerLowest = surfaceContainerLowestDark, - surfaceContainerLow = surfaceContainerLowDark, - surfaceContainer = surfaceContainerDark, - surfaceContainerHigh = surfaceContainerHighDark, - surfaceContainerHighest = surfaceContainerHighestDark, -) - -private val mediumContrastLightColorScheme = lightColorScheme( - primary = primaryLightMediumContrast, - onPrimary = onPrimaryLightMediumContrast, - primaryContainer = primaryContainerLightMediumContrast, - onPrimaryContainer = onPrimaryContainerLightMediumContrast, - secondary = secondaryLightMediumContrast, - onSecondary = onSecondaryLightMediumContrast, - secondaryContainer = secondaryContainerLightMediumContrast, - onSecondaryContainer = onSecondaryContainerLightMediumContrast, - tertiary = tertiaryLightMediumContrast, - onTertiary = onTertiaryLightMediumContrast, - tertiaryContainer = tertiaryContainerLightMediumContrast, - onTertiaryContainer = onTertiaryContainerLightMediumContrast, - error = errorLightMediumContrast, - onError = onErrorLightMediumContrast, - errorContainer = errorContainerLightMediumContrast, - onErrorContainer = onErrorContainerLightMediumContrast, - background = backgroundLightMediumContrast, - onBackground = onBackgroundLightMediumContrast, - surface = surfaceLightMediumContrast, - onSurface = onSurfaceLightMediumContrast, - surfaceVariant = surfaceVariantLightMediumContrast, - onSurfaceVariant = onSurfaceVariantLightMediumContrast, - outline = outlineLightMediumContrast, - outlineVariant = outlineVariantLightMediumContrast, - scrim = scrimLightMediumContrast, - inverseSurface = inverseSurfaceLightMediumContrast, - inverseOnSurface = inverseOnSurfaceLightMediumContrast, - inversePrimary = inversePrimaryLightMediumContrast, - surfaceDim = surfaceDimLightMediumContrast, - surfaceBright = surfaceBrightLightMediumContrast, - surfaceContainerLowest = surfaceContainerLowestLightMediumContrast, - surfaceContainerLow = surfaceContainerLowLightMediumContrast, - surfaceContainer = surfaceContainerLightMediumContrast, - surfaceContainerHigh = surfaceContainerHighLightMediumContrast, - surfaceContainerHighest = surfaceContainerHighestLightMediumContrast, -) - -private val highContrastLightColorScheme = lightColorScheme( - primary = primaryLightHighContrast, - onPrimary = onPrimaryLightHighContrast, - primaryContainer = primaryContainerLightHighContrast, - onPrimaryContainer = onPrimaryContainerLightHighContrast, - secondary = secondaryLightHighContrast, - onSecondary = onSecondaryLightHighContrast, - secondaryContainer = secondaryContainerLightHighContrast, - onSecondaryContainer = onSecondaryContainerLightHighContrast, - tertiary = tertiaryLightHighContrast, - onTertiary = onTertiaryLightHighContrast, - tertiaryContainer = tertiaryContainerLightHighContrast, - onTertiaryContainer = onTertiaryContainerLightHighContrast, - error = errorLightHighContrast, - onError = onErrorLightHighContrast, - errorContainer = errorContainerLightHighContrast, - onErrorContainer = onErrorContainerLightHighContrast, - background = backgroundLightHighContrast, - onBackground = onBackgroundLightHighContrast, - surface = surfaceLightHighContrast, - onSurface = onSurfaceLightHighContrast, - surfaceVariant = surfaceVariantLightHighContrast, - onSurfaceVariant = onSurfaceVariantLightHighContrast, - outline = outlineLightHighContrast, - outlineVariant = outlineVariantLightHighContrast, - scrim = scrimLightHighContrast, - inverseSurface = inverseSurfaceLightHighContrast, - inverseOnSurface = inverseOnSurfaceLightHighContrast, - inversePrimary = inversePrimaryLightHighContrast, - surfaceDim = surfaceDimLightHighContrast, - surfaceBright = surfaceBrightLightHighContrast, - surfaceContainerLowest = surfaceContainerLowestLightHighContrast, - surfaceContainerLow = surfaceContainerLowLightHighContrast, - surfaceContainer = surfaceContainerLightHighContrast, - surfaceContainerHigh = surfaceContainerHighLightHighContrast, - surfaceContainerHighest = surfaceContainerHighestLightHighContrast, -) - -private val mediumContrastDarkColorScheme = darkColorScheme( - primary = primaryDarkMediumContrast, - onPrimary = onPrimaryDarkMediumContrast, - primaryContainer = primaryContainerDarkMediumContrast, - onPrimaryContainer = onPrimaryContainerDarkMediumContrast, - secondary = secondaryDarkMediumContrast, - onSecondary = onSecondaryDarkMediumContrast, - secondaryContainer = secondaryContainerDarkMediumContrast, - onSecondaryContainer = onSecondaryContainerDarkMediumContrast, - tertiary = tertiaryDarkMediumContrast, - onTertiary = onTertiaryDarkMediumContrast, - tertiaryContainer = tertiaryContainerDarkMediumContrast, - onTertiaryContainer = onTertiaryContainerDarkMediumContrast, - error = errorDarkMediumContrast, - onError = onErrorDarkMediumContrast, - errorContainer = errorContainerDarkMediumContrast, - onErrorContainer = onErrorContainerDarkMediumContrast, - background = backgroundDarkMediumContrast, - onBackground = onBackgroundDarkMediumContrast, - surface = surfaceDarkMediumContrast, - onSurface = onSurfaceDarkMediumContrast, - surfaceVariant = surfaceVariantDarkMediumContrast, - onSurfaceVariant = onSurfaceVariantDarkMediumContrast, - outline = outlineDarkMediumContrast, - outlineVariant = outlineVariantDarkMediumContrast, - scrim = scrimDarkMediumContrast, - inverseSurface = inverseSurfaceDarkMediumContrast, - inverseOnSurface = inverseOnSurfaceDarkMediumContrast, - inversePrimary = inversePrimaryDarkMediumContrast, - surfaceDim = surfaceDimDarkMediumContrast, - surfaceBright = surfaceBrightDarkMediumContrast, - surfaceContainerLowest = surfaceContainerLowestDarkMediumContrast, - surfaceContainerLow = surfaceContainerLowDarkMediumContrast, - surfaceContainer = surfaceContainerDarkMediumContrast, - surfaceContainerHigh = surfaceContainerHighDarkMediumContrast, - surfaceContainerHighest = surfaceContainerHighestDarkMediumContrast, -) - -private val highContrastDarkColorScheme = darkColorScheme( - primary = primaryDarkHighContrast, - onPrimary = onPrimaryDarkHighContrast, - primaryContainer = primaryContainerDarkHighContrast, - onPrimaryContainer = onPrimaryContainerDarkHighContrast, - secondary = secondaryDarkHighContrast, - onSecondary = onSecondaryDarkHighContrast, - secondaryContainer = secondaryContainerDarkHighContrast, - onSecondaryContainer = onSecondaryContainerDarkHighContrast, - tertiary = tertiaryDarkHighContrast, - onTertiary = onTertiaryDarkHighContrast, - tertiaryContainer = tertiaryContainerDarkHighContrast, - onTertiaryContainer = onTertiaryContainerDarkHighContrast, - error = errorDarkHighContrast, - onError = onErrorDarkHighContrast, - errorContainer = errorContainerDarkHighContrast, - onErrorContainer = onErrorContainerDarkHighContrast, - background = backgroundDarkHighContrast, - onBackground = onBackgroundDarkHighContrast, - surface = surfaceDarkHighContrast, - onSurface = onSurfaceDarkHighContrast, - surfaceVariant = surfaceVariantDarkHighContrast, - onSurfaceVariant = onSurfaceVariantDarkHighContrast, - outline = outlineDarkHighContrast, - outlineVariant = outlineVariantDarkHighContrast, - scrim = scrimDarkHighContrast, - inverseSurface = inverseSurfaceDarkHighContrast, - inverseOnSurface = inverseOnSurfaceDarkHighContrast, - inversePrimary = inversePrimaryDarkHighContrast, - surfaceDim = surfaceDimDarkHighContrast, - surfaceBright = surfaceBrightDarkHighContrast, - surfaceContainerLowest = surfaceContainerLowestDarkHighContrast, - surfaceContainerLow = surfaceContainerLowDarkHighContrast, - surfaceContainer = surfaceContainerDarkHighContrast, - surfaceContainerHigh = surfaceContainerHighDarkHighContrast, - surfaceContainerHighest = surfaceContainerHighestDarkHighContrast, -) - -@Immutable -data class ColorFamily( - val color: Color, - val onColor: Color, - val colorContainer: Color, - val onColorContainer: Color -) - -val unspecified_scheme = ColorFamily( - Color.Unspecified, Color.Unspecified, Color.Unspecified, Color.Unspecified -) - -@Composable -fun AppTheme( - darkTheme: Boolean = isSystemInDarkTheme(), - // Dynamic color is available on Android 12+ - dynamicColor: Boolean = true, - content: @Composable () -> Unit -) { - val colorScheme = when { - dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> { - val context = LocalContext.current - if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) - } - - darkTheme -> darkScheme - else -> lightScheme - } - - MaterialTheme( - colorScheme = colorScheme, - typography = AppTypography, - content = content - ) -} diff --git a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Type.kt b/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Type.kt deleted file mode 100644 index 124e1b0..0000000 --- a/app/src/main/java/dev/shantoislam/agenticwebview/app/ui/theme/Type.kt +++ /dev/null @@ -1,34 +0,0 @@ -package dev.shantoislam.agenticwebview.app.ui.theme - -import androidx.compose.material3.Typography -import androidx.compose.ui.text.TextStyle -import androidx.compose.ui.text.font.FontFamily -import androidx.compose.ui.text.font.FontWeight -import androidx.compose.ui.unit.sp - -// Set of Material typography styles to start with -val AppTypography = Typography( - bodyLarge = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Normal, - fontSize = 16.sp, - lineHeight = 24.sp, - letterSpacing = 0.5.sp - ) - /* Other default text styles to override - titleLarge = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Normal, - fontSize = 22.sp, - lineHeight = 28.sp, - letterSpacing = 0.sp - ), - labelSmall = TextStyle( - fontFamily = FontFamily.Default, - fontWeight = FontWeight.Medium, - fontSize = 11.sp, - lineHeight = 16.sp, - letterSpacing = 0.5.sp - ) - */ -) \ No newline at end of file diff --git a/app/src/main/res/values/colors.xml b/app/src/main/res/values/colors.xml deleted file mode 100644 index f8c6127..0000000 --- a/app/src/main/res/values/colors.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - #FFBB86FC - #FF6200EE - #FF3700B3 - #FF03DAC5 - #FF018786 - #FF000000 - #FFFFFFFF - \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml deleted file mode 100644 index 6f91976..0000000 --- a/app/src/main/res/values/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - Webview Agent - \ No newline at end of file diff --git a/app/src/main/res/xml/backup_rules.xml b/app/src/main/res/xml/backup_rules.xml deleted file mode 100644 index 4df9255..0000000 --- a/app/src/main/res/xml/backup_rules.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - \ No newline at end of file diff --git a/app/src/main/res/xml/data_extraction_rules.xml b/app/src/main/res/xml/data_extraction_rules.xml deleted file mode 100644 index 9ee9997..0000000 --- a/app/src/main/res/xml/data_extraction_rules.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/app/src/test/java/dev/shantoislam/agenticwebview/app/ExampleUnitTest.kt b/app/src/test/java/dev/shantoislam/agenticwebview/app/ExampleUnitTest.kt deleted file mode 100644 index e7ca34c..0000000 --- a/app/src/test/java/dev/shantoislam/agenticwebview/app/ExampleUnitTest.kt +++ /dev/null @@ -1,17 +0,0 @@ -package dev.shantoislam.agenticwebview.app - -import org.junit.Test - -import org.junit.Assert.* - -/** - * Example local unit test, which will execute on the development machine (host). - * - * See [testing documentation](http://d.android.com/tools/testing). - */ -class ExampleUnitTest { - @Test - fun addition_isCorrect() { - assertEquals(4, 2 + 2) - } -} \ No newline at end of file diff --git a/browser-api/build.gradle.kts b/browser-api/build.gradle.kts new file mode 100644 index 0000000..8a81a3d --- /dev/null +++ b/browser-api/build.gradle.kts @@ -0,0 +1,29 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + `java-library` + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.maven.publish) +} + +group = "dev.shantoislam.agenticwebview" +version = providers.gradleProperty("VERSION_NAME").get() + +kotlin { + jvmToolchain(17) + compilerOptions { + jvmTarget.set(JvmTarget.JVM_11) + } +} + +dependencies { + api(libs.kotlinx.coroutines.core) + api(libs.kotlinx.serialization.json) + + testImplementation(libs.junit) + testImplementation(libs.kotlinx.coroutines.test) +} + +extra["POM_ARTIFACT_ID"] = "browser-api" +apply(from = rootProject.file("gradle/publishing.gradle.kts")) diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserCommand.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserCommand.kt new file mode 100644 index 0000000..b48fe51 --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserCommand.kt @@ -0,0 +1,141 @@ +package dev.shantoislam.agenticwebview.api + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +sealed interface BrowserCommand { + @Serializable @SerialName("click") + data class Click(val target: ElementRef, val button: PointerButton = PointerButton.PRIMARY) : BrowserCommand + + @Serializable @SerialName("long_press") + data class LongPress(val target: ElementRef, val durationMs: Long = 500) : BrowserCommand { + init { require(durationMs in 1..60_000) { "durationMs is out of range" } } + } + + @Serializable @SerialName("type_text") + data class TypeText(val target: ElementRef, val text: String, val mode: TextInputMode = TextInputMode.REPLACE_ALL) : BrowserCommand + + @Serializable @SerialName("select_option") + data class SelectOption(val target: ElementRef, val option: SelectOptionMatcher) : BrowserCommand + + @Serializable @SerialName("press_keys") + data class PressKeys(val chord: KeyChord) : BrowserCommand + + @Serializable @SerialName("scroll") + data class Scroll(val target: ScrollTarget = ScrollTarget.Page, val delta: ScrollDelta) : BrowserCommand + + @Serializable @SerialName("scroll_into_view") + data class ScrollIntoView(val target: ElementRef, val alignment: ScrollAlignment = ScrollAlignment.CENTER) : BrowserCommand +} + +@Serializable +enum class PointerButton { PRIMARY, SECONDARY, MIDDLE } + +@Serializable +enum class TextInputMode { REPLACE_ALL, APPEND, INSERT_AT_SELECTION, CLEAR } + +@Serializable +sealed interface SelectOptionMatcher { + @Serializable @SerialName("value") data class Value(val value: String) : SelectOptionMatcher + @Serializable @SerialName("label") data class Label(val label: String) : SelectOptionMatcher + @Serializable @SerialName("index") data class Index(val index: Int) : SelectOptionMatcher { + init { require(index >= 0) { "Option index must be non-negative" } } + } +} + +@Serializable +data class KeyChord( + val key: String, + val control: Boolean = false, + val alt: Boolean = false, + val shift: Boolean = false, + val meta: Boolean = false, +) { + init { require(key.isNotBlank()) { "Key must not be blank" } } +} + +@Serializable +sealed interface ScrollTarget { + @Serializable @SerialName("page") data object Page : ScrollTarget + @Serializable @SerialName("element") data class Element(val target: ElementRef) : ScrollTarget +} + +@Serializable +data class ScrollDelta(val xCssPx: Double = 0.0, val yCssPx: Double = 0.0) { + init { require(xCssPx.isFinite() && yCssPx.isFinite()) { "Scroll delta must be finite" } } +} + +@Serializable +enum class ScrollAlignment { START, CENTER, END, NEAREST } + +@Serializable +data class NavigationRequest( + val url: String, + val readiness: WaitCondition = WaitCondition.PageReady, +) { + init { require(url.isNotBlank()) { "Navigation URL must not be blank" } } +} + +@Serializable +data class NavigationReceipt( + val operation: NavigationOperation, + val requestedUrl: String? = null, + val finalUrl: String, + val documentId: DocumentId, + val phase: BrowserSessionPhase, +) + +@Serializable +data class HistoryNavigationRequest( + val operation: NavigationOperation, + val readiness: WaitCondition = WaitCondition.PageReady, +) { + init { + require(operation != NavigationOperation.URL) { + "HistoryNavigationRequest cannot use NavigationOperation.URL" + } + } +} + +@Serializable +enum class NavigationOperation { URL, BACK, FORWARD, RELOAD } + +@Serializable +data class CommandReceipt( + val commandType: String, + val strategy: ActionExecutionStrategy, + val documentId: DocumentId, + val revisionBefore: ObservationRevision, + val revisionAfter: ObservationRevision, + val dispatched: Boolean, + val verified: Boolean, + val pageChanged: Boolean, +) + +@Serializable +enum class ActionExecutionStrategy { + DOM_POINTER, + DOM_NATIVE_SETTER, + DOM_SELECT, + DOM_KEYBOARD, + DOM_SCROLL, + ANDROID_NATIVE_POINTER, +} + +@Serializable +sealed interface WaitCondition { + @Serializable @SerialName("page_ready") data object PageReady : WaitCondition + @Serializable @SerialName("dom_quiet") data class DomQuiet(val quietWindowMs: Long = 500) : WaitCondition { + init { require(quietWindowMs in 1..60_000) { "quietWindowMs is out of range" } } + } + @Serializable @SerialName("element_present") data class ElementPresent(val target: ElementRef) : WaitCondition +} + +@Serializable +data class WaitReceipt( + val conditionType: String, + val satisfiedAtEpochMs: Long, + val documentId: DocumentId, + val revision: ObservationRevision, +) diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfiguration.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfiguration.kt new file mode 100644 index 0000000..2de6ad4 --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfiguration.kt @@ -0,0 +1,182 @@ +package dev.shantoislam.agenticwebview.api + +import java.net.IDN +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient + +@Serializable +data class AgenticBrowserConfiguration( + val runtime: RuntimeConfiguration = RuntimeConfiguration(), + val observation: ObservationConfiguration = ObservationConfiguration(), + val actions: ActionConfiguration = ActionConfiguration(), + val navigation: NavigationPolicy = NavigationPolicy(), + val screenshots: ScreenshotConfiguration = ScreenshotConfiguration(), + val diagnostics: DiagnosticsConfiguration = DiagnosticsConfiguration(), + val experimental: ExperimentalConfiguration = ExperimentalConfiguration(), +) { + fun validationErrors(): List = buildList { + val allowed = navigation.allowedHosts.map { it.key }.toSet() + val denied = navigation.deniedHosts.map { it.key }.toSet() + val conflicts = allowed intersect denied + if (conflicts.isNotEmpty()) add("Host rules cannot be both allowed and denied: ${conflicts.sorted()}") + } + + fun requireValid(): AgenticBrowserConfiguration = apply { + val errors = validationErrors() + require(errors.isEmpty()) { errors.joinToString(separator = "; ") } + } +} + +@Serializable +data class RuntimeConfiguration( + val requestTimeoutMs: Long = 5_000, + val initializationTimeoutMs: Long = 10_000, + val readyQuietWindowMs: Long = 100, + val maximumPendingRequests: Int = 64, + val maximumMessageBytes: Int = 2 * 1024 * 1024, +) { + init { + require(requestTimeoutMs in 1..120_000) { "requestTimeoutMs must be within 1..120000" } + require(initializationTimeoutMs in 1..120_000) { "initializationTimeoutMs must be within 1..120000" } + require(readyQuietWindowMs in 0..10_000) { "readyQuietWindowMs must be within 0..10000" } + require(maximumPendingRequests in 1..1_024) { "maximumPendingRequests must be within 1..1024" } + require(maximumMessageBytes in 1_024..16 * 1024 * 1024) { "maximumMessageBytes must be within 1024..16777216" } + } +} + +@Serializable +data class ObservationConfiguration( + val maximumVisitedNodes: Int = 10_000, + val maximumEmittedNodes: Int = 750, + val maximumTotalTextCharacters: Int = 100_000, + val maximumTextCharactersPerNode: Int = 2_000, + val maximumTraversalMs: Long = 1_500, + val viewportExpansionPx: Int = 0, + val maximumFrameDepth: Int = 8, + val maximumShadowDepth: Int = 16, +) { + init { + require(maximumVisitedNodes in 1..1_000_000) { "maximumVisitedNodes is out of range" } + require(maximumEmittedNodes in 1..100_000) { "maximumEmittedNodes is out of range" } + require(maximumEmittedNodes <= maximumVisitedNodes) { "maximumEmittedNodes cannot exceed maximumVisitedNodes" } + require(maximumTotalTextCharacters in 1..10_000_000) { "maximumTotalTextCharacters is out of range" } + require(maximumTextCharactersPerNode in 1..100_000) { "maximumTextCharactersPerNode is out of range" } + require(maximumTraversalMs in 1..30_000) { "maximumTraversalMs is out of range" } + require(viewportExpansionPx >= -1) { "viewportExpansionPx must be -1 or non-negative" } + require(maximumFrameDepth in 0..64) { "maximumFrameDepth is out of range" } + require(maximumShadowDepth in 0..64) { "maximumShadowDepth is out of range" } + } +} + +@Serializable +data class ActionConfiguration( + val preparationTimeoutMs: Long = 3_000, + val verificationTimeoutMs: Long = 3_000, + val geometryStableCycles: Int = 2, + val geometryTolerancePx: Double = 1.0, + val pointerStrategy: PointerActionStrategy = PointerActionStrategy.DOM_ONLY, +) { + init { + require(preparationTimeoutMs in 1..120_000) { "preparationTimeoutMs is out of range" } + require(verificationTimeoutMs in 1..120_000) { "verificationTimeoutMs is out of range" } + require(geometryStableCycles in 1..20) { "geometryStableCycles is out of range" } + require(geometryTolerancePx in 0.0..100.0) { "geometryTolerancePx is out of range" } + } +} + +@Serializable +enum class PointerActionStrategy { DOM_ONLY, NATIVE_PREFERRED } + +@Serializable +data class ScreenshotConfiguration( + val maximumDimensionPx: Int = 1_920, + val jpegQuality: Int = 75, + val maximumEncodedBytes: Int = 8 * 1024 * 1024, + val maskCssSelectors: List = emptyList(), +) { + init { + require(maximumDimensionPx in 1..16_384) { "maximumDimensionPx is out of range" } + require(jpegQuality in 0..100) { "jpegQuality must be within 0..100" } + require(maximumEncodedBytes in 1_024..64 * 1024 * 1024) { "maximumEncodedBytes is out of range" } + require(maskCssSelectors.size <= 100) { "At most 100 screenshot mask selectors are allowed" } + require(maskCssSelectors.all { it.isNotBlank() && it.length <= 500 }) { + "Screenshot mask selectors must be non-blank and at most 500 characters" + } + } +} + +@Serializable +data class DiagnosticsConfiguration( + val enabled: Boolean = false, + val includePageContent: Boolean = false, +) + +@Serializable +data class ExperimentalConfiguration( + val hideWebDriverProperty: Boolean = false, + val installChromeLikeGlobals: Boolean = false, + val forceFutureShadowRootsOpen: Boolean = false, +) + +@Serializable +data class NavigationPolicy( + val allowedSchemes: Set = setOf("https"), + val allowedHosts: Set = emptySet(), + val deniedHosts: Set = emptySet(), + val popupPolicy: PrivilegedRequestPolicy = PrivilegedRequestPolicy.ASK, + val dialogPolicy: PrivilegedRequestPolicy = PrivilegedRequestPolicy.ASK, + val downloadPolicy: PrivilegedRequestPolicy = PrivilegedRequestPolicy.ASK, + val permissionPolicy: PrivilegedRequestPolicy = PrivilegedRequestPolicy.DENY, + val fileChooserPolicy: PrivilegedRequestPolicy = PrivilegedRequestPolicy.ASK, +) { + init { + require(allowedSchemes.isNotEmpty()) { "At least one URL scheme must be allowed" } + require(allowedSchemes.all { it.isNotBlank() && it == it.lowercase() }) { + "Allowed schemes must be non-blank lowercase values" + } + } +} + +@Serializable +enum class PrivilegedRequestPolicy { ALLOW, DENY, ASK } + +@Serializable +sealed interface HostRule { + val host: String + val key: String + + fun matches(candidateHost: String): Boolean + + @Serializable + @SerialName("exact") + data class Exact(override val host: String) : HostRule { + @Transient + private val normalized = normalizeHost(host) + @Transient + override val key: String = "exact:$normalized" + override fun matches(candidateHost: String): Boolean = normalizeHost(candidateHost) == normalized + } + + @Serializable + @SerialName("domain_and_subdomains") + data class DomainAndSubdomains(override val host: String) : HostRule { + @Transient + private val normalized = normalizeHost(host) + @Transient + override val key: String = "domain:$normalized" + override fun matches(candidateHost: String): Boolean { + val candidate = normalizeHost(candidateHost) + return candidate == normalized || candidate.endsWith(".$normalized") + } + } +} + +private fun normalizeHost(raw: String): String { + val trimmed = raw.trim().trimEnd('.').lowercase() + require(trimmed.isNotEmpty()) { "Host must not be blank" } + require('/' !in trimmed && ':' !in trimmed) { "Host rules must contain a hostname only" } + val ascii = IDN.toASCII(trimmed, IDN.USE_STD3_ASCII_RULES).lowercase() + require(ascii.length <= 253) { "Host is too long" } + return ascii +} diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserObservation.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserObservation.kt new file mode 100644 index 0000000..75de9a4 --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserObservation.kt @@ -0,0 +1,178 @@ +package dev.shantoislam.agenticwebview.api + +import kotlinx.serialization.Serializable + +@Serializable +data class BrowserObservation( + val id: ObservationId, + val capturedAtEpochMs: Long, + val contentTrust: ObservationContentTrust = ObservationContentTrust.UNTRUSTED_WEBPAGE, + val document: PageDocument, + val revision: ObservationRevision, + val viewport: BrowserViewport, + val frames: List, + val nodes: List, + val compactText: String, + val screenshot: BrowserScreenshot? = null, + val truncation: ObservationTruncation? = null, + val warnings: List = emptyList(), + val metrics: ObservationMetrics = ObservationMetrics(), +) { + init { require(capturedAtEpochMs >= 0) { "capturedAtEpochMs must be non-negative" } } +} + +@Serializable +enum class ObservationContentTrust { UNTRUSTED_WEBPAGE } + +@Serializable +data class PageDocument( + val id: DocumentId, + val url: String, + val title: String, + val phase: BrowserSessionPhase, +) + +@Serializable +data class BrowserViewport( + val scrollXCssPx: Double, + val scrollYCssPx: Double, + val widthCssPx: Double, + val heightCssPx: Double, + val contentWidthCssPx: Double, + val contentHeightCssPx: Double, + val devicePixelRatio: Double, + val visualViewportScale: Double, +) + +@Serializable +data class PageFrame( + val id: FrameId, + val parentId: FrameId?, + val documentId: DocumentId, + val url: String?, + val origin: String?, + val depth: Int, + val capability: FrameCapability, +) { + init { require(depth >= 0) { "Frame depth must be non-negative" } } +} + +@Serializable +enum class FrameCapability { + OBSERVABLE_AND_ACTIONABLE, + OBSERVABLE_ONLY, + INACCESSIBLE_CROSS_ORIGIN, + SANDBOX_RESTRICTED, + DETACHED, +} + +@Serializable +data class PageNode( + val nodeId: String, + val parentNodeId: String?, + val frameId: FrameId, + val depth: Int, + val kind: PageNodeKind, + val tagName: String?, + val role: String?, + val text: String?, + val accessibleName: String?, + val accessibleDescription: String?, + val attributes: Map = emptyMap(), + val states: Set = emptySet(), + val bounds: ElementBounds? = null, + val visibility: ElementVisibility = ElementVisibility.UNKNOWN, + val elementRef: ElementRef? = null, +) { + init { + require(nodeId.isNotBlank()) { "nodeId must not be blank" } + require(depth >= 0) { "Node depth must be non-negative" } + } +} + +@Serializable +enum class PageNodeKind { DOCUMENT, LANDMARK, HEADING, TEXT, LINK, CONTROL, LIST, LIST_ITEM, TABLE, ROW, CELL, IMAGE, FRAME, OTHER } + +@Serializable +enum class PageNodeState { CHECKED, SELECTED, EXPANDED, COLLAPSED, DISABLED, READ_ONLY, REQUIRED, FOCUSED, EDITABLE, MULTISELECTABLE } + +@Serializable +data class ElementBounds( + val leftCssPx: Double, + val topCssPx: Double, + val widthCssPx: Double, + val heightCssPx: Double, +) + +@Serializable +enum class ElementVisibility { VISIBLE, OFFSCREEN, OCCLUDED, HIDDEN, UNKNOWN } + +@Serializable +data class BrowserScreenshot( + val bytes: ByteArray, + val mimeType: String, + val widthPx: Int, + val heightPx: Int, +) { + init { + require(bytes.isNotEmpty()) { "Screenshot bytes must not be empty" } + require(mimeType.isNotBlank()) { "Screenshot MIME type must not be blank" } + require(widthPx > 0 && heightPx > 0) { "Screenshot dimensions must be positive" } + } + + override fun equals(other: Any?): Boolean = + other is BrowserScreenshot && + bytes.contentEquals(other.bytes) && + mimeType == other.mimeType && + widthPx == other.widthPx && + heightPx == other.heightPx + + override fun hashCode(): Int { + var result = bytes.contentHashCode() + result = 31 * result + mimeType.hashCode() + result = 31 * result + widthPx + result = 31 * result + heightPx + return result + } +} + +@Serializable +data class ObservationTruncation( + val reason: ObservationTruncationReason, + val limit: Long, + val observed: Long, +) + +@Serializable +enum class ObservationTruncationReason { VISITED_NODES, EMITTED_NODES, TOTAL_TEXT, TRAVERSAL_TIME, MESSAGE_SIZE } + +@Serializable +data class ObservationWarning(val code: String, val message: String) + +@Serializable +data class ObservationMetrics( + val durationMs: Long = 0, + val visitedNodeCount: Int = 0, + val emittedNodeCount: Int = 0, + val textCharacterCount: Int = 0, + val encodedByteCount: Int = 0, +) { + init { + require(durationMs >= 0) { "durationMs must be non-negative" } + require(visitedNodeCount >= 0 && emittedNodeCount >= 0 && textCharacterCount >= 0 && encodedByteCount >= 0) { + "Observation metrics must be non-negative" + } + } +} + +@Serializable +data class ObservationOptions( + val includeScreenshot: Boolean = false, + val includeOffscreenContent: Boolean = false, + val includeCompactText: Boolean = true, +) + +interface BrowserScreenshotProvider : AutoCloseable { + suspend fun capture(): BrowserResult + override fun close() = Unit +} diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserResult.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserResult.kt new file mode 100644 index 0000000..3a1382c --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserResult.kt @@ -0,0 +1,116 @@ +package dev.shantoislam.agenticwebview.api + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +sealed interface BrowserResult { + @Serializable + @SerialName("success") + data class Success( + val value: T, + val diagnostics: OperationDiagnostics = OperationDiagnostics(), + ) : BrowserResult + + @Serializable + @SerialName("failure") + data class Failure( + val error: BrowserError, + val diagnostics: OperationDiagnostics = OperationDiagnostics(), + ) : BrowserResult +} + +@Serializable +data class OperationDiagnostics( + val operationId: String? = null, + val durationMs: Long? = null, + val strategy: String? = null, + val attempts: Int = 1, + val warnings: List = emptyList(), +) { + init { + require(durationMs == null || durationMs >= 0) { "durationMs must be non-negative" } + require(attempts >= 1) { "attempts must be at least 1" } + } +} + +@Serializable +sealed interface BrowserError { + val message: String + + @Serializable @SerialName("session_not_attached") + data class SessionNotAttached(override val message: String = "Browser session is not attached") : BrowserError + + @Serializable @SerialName("session_closed") + data class SessionClosed(override val message: String = "Browser session is closed") : BrowserError + + @Serializable @SerialName("navigation_blocked") + data class NavigationBlocked(val url: String, val reason: String, override val message: String = "Navigation was blocked") : BrowserError + + @Serializable @SerialName("navigation_failed") + data class NavigationFailed(val url: String, val httpStatus: Int? = null, override val message: String = "Navigation failed") : BrowserError + + @Serializable @SerialName("navigation_history_unavailable") + data class NavigationHistoryUnavailable( + val operation: NavigationOperation, + override val message: String = "Requested browser history navigation is unavailable", + ) : BrowserError + + @Serializable @SerialName("page_not_ready") + data class PageNotReady(val phase: BrowserSessionPhase, override val message: String = "Page is not ready") : BrowserError + + @Serializable @SerialName("stale_element") + data class StaleElementReference(val target: ElementRef, override val message: String = "Element reference is stale") : BrowserError + + @Serializable @SerialName("element_not_found") + data class ElementNotFound(val target: ElementRef, override val message: String = "Element was not found") : BrowserError + + @Serializable @SerialName("element_not_actionable") + data class ElementNotActionable(val target: ElementRef, val reason: String, override val message: String = "Element is not actionable") : BrowserError + + @Serializable @SerialName("element_occluded") + data class ElementOccluded(val target: ElementRef, val occludedBy: ElementRef? = null, override val message: String = "Element is occluded") : BrowserError + + @Serializable @SerialName("unsupported_frame") + data class UnsupportedFrame(val frameId: FrameId, val reason: String, override val message: String = "Frame is not supported") : BrowserError + + @Serializable @SerialName("unsupported_action") + data class UnsupportedAction(val action: String, val reason: String, override val message: String = "Action is not supported") : BrowserError + + @Serializable @SerialName("action_rejected") + data class ActionRejected(val action: String, val reason: String, override val message: String = "Action was rejected") : BrowserError + + @Serializable @SerialName("action_not_verified") + data class ActionNotVerified( + val action: String, + val receipt: CommandReceipt? = null, + override val message: String = "Action was dispatched but its effect could not be verified", + ) : BrowserError + + @Serializable @SerialName("runtime_unavailable") + data class RuntimeUnavailable(override val message: String = "Page runtime is unavailable") : BrowserError + + @Serializable @SerialName("protocol_mismatch") + data class RuntimeProtocolMismatch(val expected: Int, val actual: Int?, override val message: String = "Runtime protocol version mismatch") : BrowserError + + @Serializable @SerialName("runtime_failure") + data class RuntimeFailure(val code: String, override val message: String, val details: Map = emptyMap()) : BrowserError + + @Serializable @SerialName("malformed_runtime_response") + data class MalformedRuntimeResponse(override val message: String) : BrowserError + + @Serializable @SerialName("renderer_terminated") + data class RendererTerminated(val didCrash: Boolean, override val message: String = "WebView renderer terminated") : BrowserError + + @Serializable @SerialName("screenshot_failure") + data class ScreenshotFailure(override val message: String) : BrowserError + + @Serializable @SerialName("timeout") + data class Timeout(val operation: String, val timeoutMs: Long, override val message: String = "Operation timed out") : BrowserError + + @Serializable @SerialName("cancelled") + data class Cancelled(val operation: String, override val message: String = "Operation was cancelled") : BrowserError + + @Serializable @SerialName("resource_limit") + data class ResourceLimitExceeded(val resource: String, val limit: Long, override val message: String = "Resource limit exceeded") : BrowserError +} diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserSession.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserSession.kt new file mode 100644 index 0000000..9d8ea59 --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/BrowserSession.kt @@ -0,0 +1,94 @@ +package dev.shantoislam.agenticwebview.api + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +interface AgenticBrowserSession : AutoCloseable { + val state: StateFlow + val events: Flow + + suspend fun navigate(request: NavigationRequest): BrowserResult + suspend fun navigateHistory(request: HistoryNavigationRequest): BrowserResult + suspend fun observe(options: ObservationOptions = ObservationOptions()): BrowserResult + suspend fun execute(command: BrowserCommand): BrowserResult + suspend fun await(condition: WaitCondition): BrowserResult + + override fun close() +} + +@Serializable +data class BrowserSessionState( + val phase: BrowserSessionPhase, + val documentId: DocumentId? = null, + val revision: ObservationRevision = ObservationRevision(0), + val url: String? = null, +) + +@Serializable +enum class BrowserSessionPhase { + DETACHED, + ATTACHED, + NAVIGATING, + DOCUMENT_CREATED, + RUNTIME_INITIALIZING, + INTERACTIVE, + STABILIZING, + READY, + FAILED, + RENDERER_TERMINATED, + CLOSED, +} + +@Serializable +sealed interface BrowserEvent { + @Serializable @SerialName("state_changed") + data class StateChanged(val previous: BrowserSessionState, val current: BrowserSessionState) : BrowserEvent + + @Serializable @SerialName("document_revision") + data class DocumentRevisionChanged(val documentId: DocumentId, val revision: ObservationRevision) : BrowserEvent + + @Serializable @SerialName("popup_requested") + data class PopupRequested(val url: String?, val isUserGesture: Boolean) : BrowserEvent + + @Serializable @SerialName("dialog_requested") + data class DialogRequested(val type: DialogType, val message: String, val defaultValue: String? = null) : BrowserEvent + + @Serializable @SerialName("download_requested") + data class DownloadRequested(val url: String, val mimeType: String?, val contentLength: Long?) : BrowserEvent + + @Serializable @SerialName("permission_requested") + data class PermissionRequested(val origin: String, val resources: List) : BrowserEvent + + @Serializable @SerialName("file_chooser_requested") + data class FileChooserRequested(val acceptTypes: List, val captureEnabled: Boolean) : BrowserEvent + + @Serializable @SerialName("navigation_blocked") + data class NavigationBlocked(val url: String, val reason: String) : BrowserEvent + + @Serializable @SerialName("ssl_error") + data class SslErrorReceived(val url: String, val primaryError: Int) : BrowserEvent + + @Serializable @SerialName("safe_browsing_hit") + data class SafeBrowsingHit(val url: String, val threatType: Int) : BrowserEvent + + @Serializable @SerialName("renderer_terminated") + data class RendererTerminated(val didCrash: Boolean) : BrowserEvent +} + +@Serializable +enum class DialogType { ALERT, CONFIRM, PROMPT, BEFORE_UNLOAD } + +fun interface BrowserDiagnosticsSink { + fun emit(event: BrowserDiagnosticEvent) +} + +data class BrowserDiagnosticEvent( + val category: String, + val operationId: String? = null, + val documentId: DocumentId? = null, + val revision: ObservationRevision? = null, + val durationMs: Long? = null, + val attributes: Map = emptyMap(), +) diff --git a/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/Identifiers.kt b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/Identifiers.kt new file mode 100644 index 0000000..25c9f1b --- /dev/null +++ b/browser-api/src/main/kotlin/dev/shantoislam/agenticwebview/api/Identifiers.kt @@ -0,0 +1,45 @@ +package dev.shantoislam.agenticwebview.api + +import kotlinx.serialization.Serializable + +@Serializable +@JvmInline +value class DocumentId(val value: String) { + init { require(value.isNotBlank()) { "DocumentId must not be blank" } } +} + +@Serializable +@JvmInline +value class FrameId(val value: String) { + init { require(value.isNotBlank()) { "FrameId must not be blank" } } + + companion object { + val Main = FrameId("main") + } +} + +@Serializable +@JvmInline +value class ElementId(val value: String) { + init { require(value.isNotBlank()) { "ElementId must not be blank" } } +} + +@Serializable +@JvmInline +value class ObservationId(val value: String) { + init { require(value.isNotBlank()) { "ObservationId must not be blank" } } +} + +@Serializable +@JvmInline +value class ObservationRevision(val value: Long) { + init { require(value >= 0) { "ObservationRevision must be non-negative" } } +} + +@Serializable +data class ElementRef( + val documentId: DocumentId, + val frameId: FrameId, + val elementId: ElementId, + val observedAtRevision: ObservationRevision, +) diff --git a/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfigurationTest.kt b/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfigurationTest.kt new file mode 100644 index 0000000..62ecd73 --- /dev/null +++ b/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/BrowserConfigurationTest.kt @@ -0,0 +1,59 @@ +package dev.shantoislam.agenticwebview.api + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test + +class BrowserConfigurationTest { + @Test + fun domainRuleMatchesOnlyHostBoundary() { + val rule = HostRule.DomainAndSubdomains("Example.COM.") + + assertTrue(rule.matches("example.com")) + assertTrue(rule.matches("a.example.com")) + assertFalse(rule.matches("evilexample.com")) + assertFalse(rule.matches("example.com.evil.test")) + } + + @Test + fun exactRuleDoesNotMatchSubdomains() { + val rule = HostRule.Exact("example.com") + + assertTrue(rule.matches("EXAMPLE.COM")) + assertFalse(rule.matches("www.example.com")) + } + + @Test + fun conflictingRulesFailValidation() { + val configuration = AgenticBrowserConfiguration( + navigation = NavigationPolicy( + allowedHosts = setOf(HostRule.Exact("example.com")), + deniedHosts = setOf(HostRule.Exact("example.com")), + ), + ) + + assertThrows(IllegalArgumentException::class.java) { configuration.requireValid() } + } + + @Test + fun invalidResourceLimitsFailAtConstruction() { + assertThrows(IllegalArgumentException::class.java) { + RuntimeConfiguration(maximumPendingRequests = 0) + } + } + + @Test + fun invalidScreenshotMaskSelectorsFailAtConstruction() { + assertThrows(IllegalArgumentException::class.java) { + ScreenshotConfiguration(maskCssSelectors = listOf(" ")) + } + } + + @Test + fun historyRequestCannotMasqueradeAsUrlNavigation() { + assertThrows(IllegalArgumentException::class.java) { + HistoryNavigationRequest(NavigationOperation.URL) + } + } +} diff --git a/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/IdentifiersTest.kt b/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/IdentifiersTest.kt new file mode 100644 index 0000000..eb89db1 --- /dev/null +++ b/browser-api/src/test/kotlin/dev/shantoislam/agenticwebview/api/IdentifiersTest.kt @@ -0,0 +1,24 @@ +package dev.shantoislam.agenticwebview.api + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertThrows +import org.junit.Test + +class IdentifiersTest { + @Test + fun elementReferenceIsScopedByDocumentAndFrame() { + val first = ElementRef(DocumentId("doc-1"), FrameId.Main, ElementId("element-1"), ObservationRevision(4)) + val second = first.copy(documentId = DocumentId("doc-2")) + + assertNotEquals(first, second) + assertEquals("element-1", first.elementId.value) + } + + @Test + fun identifiersRejectBlankValues() { + assertThrows(IllegalArgumentException::class.java) { DocumentId(" ") } + assertThrows(IllegalArgumentException::class.java) { FrameId("") } + assertThrows(IllegalArgumentException::class.java) { ElementId("\t") } + } +} diff --git a/browser-compose/build.gradle.kts b/browser-compose/build.gradle.kts new file mode 100644 index 0000000..8b85b7f --- /dev/null +++ b/browser-compose/build.gradle.kts @@ -0,0 +1,49 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.kotlin.android) + alias(libs.plugins.kotlin.compose) + alias(libs.plugins.maven.publish) +} + +group = "dev.shantoislam.agenticwebview" +version = providers.gradleProperty("VERSION_NAME").get() + +kotlin { + compilerOptions { jvmTarget.set(JvmTarget.JVM_11) } +} + +android { + namespace = "dev.shantoislam.agenticwebview.compose" + compileSdk = 37 + + defaultConfig { + minSdk = 28 + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + buildFeatures { + compose = true + } +} + +dependencies { + api(project(":browser-webview")) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.compose.ui) + implementation(libs.androidx.lifecycle.runtime.ktx) +} + +extra["POM_ARTIFACT_ID"] = "browser-compose" +apply(from = rootProject.file("gradle/publishing.gradle.kts")) diff --git a/browser-compose/src/main/AndroidManifest.xml b/browser-compose/src/main/AndroidManifest.xml new file mode 100644 index 0000000..cc947c5 --- /dev/null +++ b/browser-compose/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/browser-compose/src/main/kotlin/dev/shantoislam/agenticwebview/compose/AgenticBrowserCompose.kt b/browser-compose/src/main/kotlin/dev/shantoislam/agenticwebview/compose/AgenticBrowserCompose.kt new file mode 100644 index 0000000..ee007e7 --- /dev/null +++ b/browser-compose/src/main/kotlin/dev/shantoislam/agenticwebview/compose/AgenticBrowserCompose.kt @@ -0,0 +1,68 @@ +package dev.shantoislam.agenticwebview.compose + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalView +import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleEventObserver +import androidx.lifecycle.findViewTreeLifecycleOwner +import dev.shantoislam.agenticwebview.api.AgenticBrowserConfiguration +import dev.shantoislam.agenticwebview.api.BrowserDiagnosticsSink +import dev.shantoislam.agenticwebview.webview.AgenticBrowserHost +import dev.shantoislam.agenticwebview.webview.BrowserHostDelegate + +/** + * Creates and owns an [AgenticBrowserHost] for the lifetime of this composition. + * + * A configuration change intentionally creates a fresh browser session. This avoids carrying + * runtime policy and protocol state across incompatible configurations. + */ +@Composable +fun rememberAgenticBrowserHost( + configuration: AgenticBrowserConfiguration = AgenticBrowserConfiguration(), + delegate: BrowserHostDelegate = BrowserHostDelegate.DenyAll, + diagnosticsSink: BrowserDiagnosticsSink? = null, +): AgenticBrowserHost { + val context = LocalContext.current + val host = remember(context, configuration, delegate, diagnosticsSink) { + AgenticBrowserHost.create(context, configuration, delegate, diagnosticsSink) + } + DisposableEffect(host) { + onDispose(host::close) + } + return host +} + +/** Places an existing [AgenticBrowserHost] in the Compose UI hierarchy. */ +@Composable +fun AgenticBrowserView( + host: AgenticBrowserHost, + modifier: Modifier = Modifier, +) { + val lifecycleOwner = LocalView.current.findViewTreeLifecycleOwner() + DisposableEffect(host, lifecycleOwner) { + val observer = LifecycleEventObserver { _, event -> + when (event) { + Lifecycle.Event.ON_RESUME -> host.onResume() + Lifecycle.Event.ON_PAUSE -> host.onPause() + else -> Unit + } + } + lifecycleOwner?.lifecycle?.addObserver(observer) + if (lifecycleOwner?.lifecycle?.currentState?.isAtLeast(Lifecycle.State.RESUMED) == true) { + host.onResume() + } + onDispose { + lifecycleOwner?.lifecycle?.removeObserver(observer) + host.onPause() + } + } + AndroidView( + factory = { host.view }, + modifier = modifier, + ) +} diff --git a/browser-webview/build.gradle.kts b/browser-webview/build.gradle.kts new file mode 100644 index 0000000..a61774c --- /dev/null +++ b/browser-webview/build.gradle.kts @@ -0,0 +1,62 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + alias(libs.plugins.android.library) + alias(libs.plugins.kotlin.android) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.maven.publish) +} + +group = "dev.shantoislam.agenticwebview" +version = providers.gradleProperty("VERSION_NAME").get() + +kotlin { + compilerOptions { jvmTarget.set(JvmTarget.JVM_11) } +} + +android { + namespace = "dev.shantoislam.agenticwebview.webview" + compileSdk = 37 + + defaultConfig { + minSdk = 28 + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + + sourceSets { + getByName("main").assets.srcDir("${rootProject.projectDir}/web-runtime/dist") + getByName("test").resources.srcDir("${rootProject.projectDir}/protocol-fixtures") + } +} + +dependencies { + api(project(":browser-api")) + implementation(libs.androidx.webkit) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.serialization.json) + + testImplementation(libs.junit) + testImplementation(libs.kotlinx.coroutines.test) + androidTestImplementation(libs.androidx.junit) + androidTestImplementation(libs.kotlinx.coroutines.test) + androidTestImplementation(libs.mockwebserver) +} + +tasks.named("preBuild") { + dependsOn(rootProject.tasks.named("buildWebRuntime")) +} + +extra["POM_ARTIFACT_ID"] = "browser-webview" +apply(from = rootProject.file("gradle/publishing.gradle.kts")) diff --git a/browser-webview/consumer-rules.pro b/browser-webview/consumer-rules.pro new file mode 100644 index 0000000..7cf13b6 --- /dev/null +++ b/browser-webview/consumer-rules.pro @@ -0,0 +1,6 @@ +# Preserve the narrow JavaScript response bridge used by the versioned runtime protocol. +-keepclassmembers class dev.shantoislam.agenticwebview.webview.protocol.RuntimeProtocolBridge { + @android.webkit.JavascriptInterface ; +} + +-keepattributes RuntimeVisibleAnnotations,AnnotationDefault diff --git a/browser-webview/src/androidTest/AndroidManifest.xml b/browser-webview/src/androidTest/AndroidManifest.xml new file mode 100644 index 0000000..36f3d2c --- /dev/null +++ b/browser-webview/src/androidTest/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/browser-webview/src/androidTest/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHostInstrumentedTest.kt b/browser-webview/src/androidTest/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHostInstrumentedTest.kt new file mode 100644 index 0000000..442eb6e --- /dev/null +++ b/browser-webview/src/androidTest/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHostInstrumentedTest.kt @@ -0,0 +1,59 @@ +package dev.shantoislam.agenticwebview.webview + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import dev.shantoislam.agenticwebview.api.AgenticBrowserConfiguration +import dev.shantoislam.agenticwebview.api.BrowserResult +import dev.shantoislam.agenticwebview.api.HostRule +import dev.shantoislam.agenticwebview.api.NavigationPolicy +import dev.shantoislam.agenticwebview.api.NavigationRequest +import dev.shantoislam.agenticwebview.api.RuntimeConfiguration +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class AgenticBrowserHostInstrumentedTest { + @Test + fun navigatesInitializesRuntimeAndObservesAnActionableElement() = runBlocking { + val server = MockWebServer() + server.enqueue( + MockResponse().setHeader("Content-Type", "text/html; charset=utf-8").setBody( + "Fixture

Ready

", + ), + ) + server.start() + val instrumentation = InstrumentationRegistry.getInstrumentation() + lateinit var host: AgenticBrowserHost + instrumentation.runOnMainSync { + host = AgenticBrowserHost.create( + instrumentation.targetContext, + AgenticBrowserConfiguration( + runtime = RuntimeConfiguration(requestTimeoutMs = 10_000, initializationTimeoutMs = 10_000), + navigation = NavigationPolicy( + allowedSchemes = setOf("http"), + allowedHosts = setOf(HostRule.Exact(server.hostName)), + ), + ), + ) + } + + try { + val navigation = withTimeout(15_000) { + host.session.navigate(NavigationRequest(server.url("/").toString())) + } + assertTrue(navigation is BrowserResult.Success) + val observation = host.session.observe() + assertTrue(observation is BrowserResult.Success) + val value = (observation as BrowserResult.Success).value + assertTrue(value.nodes.any { it.tagName == "button" && it.elementRef != null }) + } finally { + instrumentation.runOnMainSync { host.close() } + server.shutdown() + } + } +} diff --git a/browser-webview/src/main/AndroidManifest.xml b/browser-webview/src/main/AndroidManifest.xml new file mode 100644 index 0000000..fbb2ef8 --- /dev/null +++ b/browser-webview/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHost.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHost.kt new file mode 100644 index 0000000..46b06fe --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AgenticBrowserHost.kt @@ -0,0 +1,81 @@ +package dev.shantoislam.agenticwebview.webview + +import android.content.Context +import android.os.Looper +import android.webkit.WebView +import dev.shantoislam.agenticwebview.api.AgenticBrowserConfiguration +import dev.shantoislam.agenticwebview.api.AgenticBrowserSession +import dev.shantoislam.agenticwebview.api.BrowserDiagnosticsSink +import dev.shantoislam.agenticwebview.api.BrowserScreenshotProvider + +class AgenticBrowserHost private constructor( + val view: WebView, + val session: AgenticBrowserSession, +) : AutoCloseable { + + override fun close() { + session.close() + } + + fun onResume() { + view.onResume() + } + + fun onPause() { + view.onPause() + } + + companion object { + @JvmStatic + fun create( + context: Context, + configuration: AgenticBrowserConfiguration = AgenticBrowserConfiguration(), + delegate: BrowserHostDelegate = BrowserHostDelegate.DenyAll, + diagnosticsSink: BrowserDiagnosticsSink? = null, + screenshotProvider: BrowserScreenshotProvider? = null, + ): AgenticBrowserHost { + check(Looper.myLooper() == Looper.getMainLooper()) { + "AgenticBrowserHost.create must be called on the Android main thread" + } + configuration.requireValid() + val webView = WebView(context) + return try { + val session = AndroidAgenticBrowserSession( + webView, + configuration, + ownsWebView = true, + delegate = delegate, + diagnosticsSink = diagnosticsSink, + screenshotProvider = screenshotProvider, + ) + AgenticBrowserHost(webView, session) + } catch (error: Exception) { + webView.destroy() + throw error + } + } + + @JvmStatic + fun attach( + webView: WebView, + configuration: AgenticBrowserConfiguration = AgenticBrowserConfiguration(), + delegate: BrowserHostDelegate = BrowserHostDelegate.DenyAll, + diagnosticsSink: BrowserDiagnosticsSink? = null, + screenshotProvider: BrowserScreenshotProvider? = null, + ): AgenticBrowserHost { + check(Looper.myLooper() == Looper.getMainLooper()) { + "AgenticBrowserHost.attach must be called on the Android main thread" + } + configuration.requireValid() + val session = AndroidAgenticBrowserSession( + webView, + configuration, + ownsWebView = false, + delegate = delegate, + diagnosticsSink = diagnosticsSink, + screenshotProvider = screenshotProvider, + ) + return AgenticBrowserHost(webView, session) + } + } +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AndroidAgenticBrowserSession.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AndroidAgenticBrowserSession.kt new file mode 100644 index 0000000..84acfc5 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/AndroidAgenticBrowserSession.kt @@ -0,0 +1,1325 @@ +package dev.shantoislam.agenticwebview.webview + +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.net.Uri +import android.os.Looper +import android.os.Message +import android.os.SystemClock +import android.net.http.SslError +import android.webkit.PermissionRequest +import android.webkit.GeolocationPermissions +import android.webkit.SafeBrowsingResponse +import android.webkit.SslErrorHandler +import android.webkit.ValueCallback +import android.webkit.JsPromptResult +import android.webkit.JsResult +import android.webkit.RenderProcessGoneDetail +import android.webkit.WebChromeClient +import android.webkit.WebResourceError +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebSettings +import android.webkit.WebView +import android.webkit.WebViewClient +import android.view.MotionEvent +import androidx.webkit.ScriptHandler +import androidx.webkit.WebViewCompat +import androidx.webkit.WebViewFeature +import dev.shantoislam.agenticwebview.api.AgenticBrowserConfiguration +import dev.shantoislam.agenticwebview.api.AgenticBrowserSession +import dev.shantoislam.agenticwebview.api.BrowserCommand +import dev.shantoislam.agenticwebview.api.BrowserDiagnosticEvent +import dev.shantoislam.agenticwebview.api.BrowserDiagnosticsSink +import dev.shantoislam.agenticwebview.api.BrowserError +import dev.shantoislam.agenticwebview.api.BrowserEvent +import dev.shantoislam.agenticwebview.api.BrowserObservation +import dev.shantoislam.agenticwebview.api.BrowserResult +import dev.shantoislam.agenticwebview.api.BrowserScreenshotProvider +import dev.shantoislam.agenticwebview.api.BrowserSessionPhase +import dev.shantoislam.agenticwebview.api.BrowserSessionState +import dev.shantoislam.agenticwebview.api.CommandReceipt +import dev.shantoislam.agenticwebview.api.DialogType +import dev.shantoislam.agenticwebview.api.DocumentId +import dev.shantoislam.agenticwebview.api.HistoryNavigationRequest +import dev.shantoislam.agenticwebview.api.NavigationOperation +import dev.shantoislam.agenticwebview.api.NavigationReceipt +import dev.shantoislam.agenticwebview.api.NavigationRequest +import dev.shantoislam.agenticwebview.api.ObservationOptions +import dev.shantoislam.agenticwebview.api.ObservationRevision +import dev.shantoislam.agenticwebview.api.ObservationWarning +import dev.shantoislam.agenticwebview.api.PointerActionStrategy +import dev.shantoislam.agenticwebview.api.PointerButton +import dev.shantoislam.agenticwebview.api.PrivilegedRequestPolicy +import dev.shantoislam.agenticwebview.api.WaitCondition +import dev.shantoislam.agenticwebview.api.WaitReceipt +import dev.shantoislam.agenticwebview.webview.protocol.GatewayResult +import dev.shantoislam.agenticwebview.webview.protocol.RuntimeMethods +import dev.shantoislam.agenticwebview.webview.protocol.RuntimeProtocolBridge +import dev.shantoislam.agenticwebview.webview.protocol.RuntimeProtocolGateway +import dev.shantoislam.agenticwebview.webview.protocol.RuntimeResponseStatus +import dev.shantoislam.agenticwebview.webview.protocol.WebViewRuntimeTransport +import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.SerializationException +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.encodeToJsonElement +import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonObject + +@SuppressLint("SetJavaScriptEnabled") +internal class AndroidAgenticBrowserSession( + private val webView: WebView, + private val configuration: AgenticBrowserConfiguration, + private val ownsWebView: Boolean, + private val delegate: BrowserHostDelegate, + private val diagnosticsSink: BrowserDiagnosticsSink?, + screenshotProvider: BrowserScreenshotProvider?, +) : AgenticBrowserSession { + private val previousWebViewClient = webView.webViewClient + private val previousWebChromeClient = webView.webChromeClient + private val scopeJob = SupervisorJob() + private val scope = CoroutineScope(scopeJob + Dispatchers.Main.immediate) + private val closed = AtomicBoolean(false) + private val sessionId = UUID.randomUUID().toString() + private val bridgeToken = UUID.randomUUID().toString() + UUID.randomUUID().toString() + private var activeDocumentId: DocumentId? = null + private var lastHttpStatus: Int? = null + private var runtimeScript: String? = null + private var documentStartScriptHandler: ScriptHandler? = null + private var lastRendererDidCrash = false + @Volatile private var navigationAborted = false + private val interactionMutex = Mutex() + + private val json = Json { + ignoreUnknownKeys = false + explicitNulls = false + encodeDefaults = true + classDiscriminator = "type" + } + private val screenshotProvider: BrowserScreenshotProvider = screenshotProvider + ?: PixelCopyScreenshotProvider( + webViewProvider = { if (closed.get()) null else webView }, + configuration = configuration.screenshots, + ) + + private val _state = MutableStateFlow(BrowserSessionState(BrowserSessionPhase.DETACHED)) + override val state: StateFlow = _state.asStateFlow() + + private val _events = MutableSharedFlow(extraBufferCapacity = 64) + override val events: Flow = _events.asSharedFlow() + + private lateinit var gateway: RuntimeProtocolGateway + private val bridge = RuntimeProtocolBridge( + scope = scope, + gatewayProvider = { if (::gateway.isInitialized) gateway else null }, + maximumMessageBytes = configuration.runtime.maximumMessageBytes, + maximumConcurrentCallbacks = configuration.runtime.maximumPendingRequests * 2, + onRejectedResponse = { emitDiagnostic("Rejected runtime response", mapOf("reason" to it)) }, + transportErrorHandler = { emitDiagnostic("Runtime transport error", mapOf("reason" to it)) }, + ) + + init { + check(Looper.myLooper() == Looper.getMainLooper()) { + "AndroidAgenticBrowserSession must be created on the Android main thread" + } + configuration.requireValid() + configureWebView() + installDocumentStartRuntime() + gateway = RuntimeProtocolGateway( + bridgeToken = bridgeToken, + maximumPendingRequests = configuration.runtime.maximumPendingRequests, + maximumMessageBytes = configuration.runtime.maximumMessageBytes, + requestTimeoutMs = configuration.runtime.requestTimeoutMs, + transport = WebViewRuntimeTransport { if (closed.get()) null else webView }, + ) + webView.addJavascriptInterface(bridge, PROTOCOL_BRIDGE_NAME) + installClients() + transition(BrowserSessionState(BrowserSessionPhase.ATTACHED)) + } + + override suspend fun navigate(request: NavigationRequest): BrowserResult = + diagnosticOperation("navigation.url") { + interactionMutex.withLock { navigateLocked(request) } + } + + private suspend fun navigateLocked(request: NavigationRequest): BrowserResult { + if (closed.get()) return failure(BrowserError.SessionClosed()) + when (val decision = NavigationPolicyEvaluator.evaluate(request.url, configuration.navigation)) { + NavigationDecision.Allow -> Unit + is NavigationDecision.Block -> { + _events.tryEmit(BrowserEvent.NavigationBlocked(request.url, decision.reason)) + safelyDelegate { onNavigationBlocked(request.url, decision.reason) } + return failure(BrowserError.NavigationBlocked(request.url, decision.reason)) + } + } + + return try { + activeDocumentId?.let { documentId -> + gateway.cancelDocument(documentId, "Navigation requested") + } + activeDocumentId = null + withContext(Dispatchers.Main.immediate) { + navigationAborted = false + transition( + BrowserSessionState( + phase = BrowserSessionPhase.NAVIGATING, + url = request.url, + ), + ) + webView.loadUrl(request.url) + } + val readyState = withTimeout(configuration.runtime.initializationTimeoutMs) { + state.first { + it.phase == BrowserSessionPhase.READY || + it.phase == BrowserSessionPhase.FAILED || + it.phase == BrowserSessionPhase.RENDERER_TERMINATED || + it.phase == BrowserSessionPhase.CLOSED + } + } + when (readyState.phase) { + BrowserSessionPhase.READY -> { + when (val waited = await(request.readiness)) { + is BrowserResult.Failure -> waited + is BrowserResult.Success -> BrowserResult.Success( + NavigationReceipt( + operation = NavigationOperation.URL, + requestedUrl = request.url, + finalUrl = readyState.url ?: request.url, + documentId = requireNotNull(readyState.documentId), + phase = readyState.phase, + ), + ) + } + } + BrowserSessionPhase.RENDERER_TERMINATED -> failure(BrowserError.RendererTerminated(lastRendererDidCrash)) + BrowserSessionPhase.CLOSED -> failure(BrowserError.SessionClosed()) + else -> failure(BrowserError.NavigationFailed(request.url, lastHttpStatus)) + } + } catch (_: kotlinx.coroutines.TimeoutCancellationException) { + abortNavigation("Navigation timed out") + failure(BrowserError.Timeout("navigation", configuration.runtime.initializationTimeoutMs)) + } catch (cancellation: CancellationException) { + withContext(NonCancellable) { + abortNavigation("Navigation caller was cancelled") + } + throw cancellation + } catch (error: Exception) { + abortNavigation("Navigation failed") + failure(BrowserError.NavigationFailed(request.url, lastHttpStatus, error.message ?: "Navigation failed")) + } + } + + override suspend fun navigateHistory( + request: HistoryNavigationRequest, + ): BrowserResult = diagnosticOperation("navigation.${request.operation.name.lowercase()}") { + interactionMutex.withLock { navigateHistoryLocked(request) } + } + + private suspend fun navigateHistoryLocked( + request: HistoryNavigationRequest, + ): BrowserResult { + if (closed.get()) return failure(BrowserError.SessionClosed()) + + val historyAvailable = withContext(Dispatchers.Main.immediate) { + when (request.operation) { + NavigationOperation.BACK -> webView.canGoBack() + NavigationOperation.FORWARD -> webView.canGoForward() + NavigationOperation.RELOAD -> webView.url != null + NavigationOperation.URL -> false + } + } + if (!historyAvailable) { + return failure(BrowserError.NavigationHistoryUnavailable(request.operation)) + } + + return try { + activeDocumentId?.let { documentId -> + gateway.cancelDocument(documentId, "History navigation requested") + } + activeDocumentId = null + withContext(Dispatchers.Main.immediate) { + navigationAborted = false + transition( + BrowserSessionState( + phase = BrowserSessionPhase.NAVIGATING, + url = webView.url, + ), + ) + when (request.operation) { + NavigationOperation.BACK -> webView.goBack() + NavigationOperation.FORWARD -> webView.goForward() + NavigationOperation.RELOAD -> webView.reload() + NavigationOperation.URL -> error("URL navigation cannot be used as a history operation") + } + } + val readyState = withTimeout(configuration.runtime.initializationTimeoutMs) { + state.first { + it.phase == BrowserSessionPhase.READY || + it.phase == BrowserSessionPhase.FAILED || + it.phase == BrowserSessionPhase.RENDERER_TERMINATED || + it.phase == BrowserSessionPhase.CLOSED + } + } + when (readyState.phase) { + BrowserSessionPhase.READY -> when (val waited = await(request.readiness)) { + is BrowserResult.Failure -> waited + is BrowserResult.Success -> BrowserResult.Success( + NavigationReceipt( + operation = request.operation, + finalUrl = readyState.url.orEmpty(), + documentId = requireNotNull(readyState.documentId), + phase = readyState.phase, + ), + ) + } + BrowserSessionPhase.RENDERER_TERMINATED -> failure(BrowserError.RendererTerminated(lastRendererDidCrash)) + BrowserSessionPhase.CLOSED -> failure(BrowserError.SessionClosed()) + else -> failure(BrowserError.NavigationFailed(readyState.url.orEmpty(), lastHttpStatus)) + } + } catch (_: kotlinx.coroutines.TimeoutCancellationException) { + abortNavigation("History navigation timed out") + failure(BrowserError.Timeout("history navigation", configuration.runtime.initializationTimeoutMs)) + } catch (cancellation: CancellationException) { + withContext(NonCancellable) { + abortNavigation("History navigation caller was cancelled") + } + throw cancellation + } catch (error: Exception) { + abortNavigation("History navigation failed") + failure( + BrowserError.NavigationFailed( + _state.value.url.orEmpty(), + lastHttpStatus, + error.message ?: "History navigation failed", + ), + ) + } + } + + private suspend fun abortNavigation(reason: String) { + navigationAborted = true + activeDocumentId?.let { gateway.cancelDocument(it, reason) } + activeDocumentId = null + withContext(Dispatchers.Main.immediate) { + webView.stopLoading() + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED, documentId = null)) + } + } + + override suspend fun observe(options: ObservationOptions): BrowserResult = + diagnosticOperation("observation.capture") { observeInternal(options) } + + private suspend fun observeInternal(options: ObservationOptions): BrowserResult { + val documentId = activeDocumentId + ?: return failure(BrowserError.PageNotReady(_state.value.phase)) + if (closed.get()) return failure(BrowserError.SessionClosed()) + if (_state.value.phase !in OBSERVABLE_PHASES) { + return failure(BrowserError.PageNotReady(_state.value.phase)) + } + + val observationConfig = configuration.observation + val payload = buildJsonObject { + put("maximumVisitedNodes", observationConfig.maximumVisitedNodes) + put("maximumEmittedNodes", observationConfig.maximumEmittedNodes) + put("maximumTotalTextCharacters", observationConfig.maximumTotalTextCharacters) + put("maximumTextCharactersPerNode", observationConfig.maximumTextCharactersPerNode) + put("maximumTraversalMs", observationConfig.maximumTraversalMs) + put("maximumFrameDepth", observationConfig.maximumFrameDepth) + put("maximumShadowDepth", observationConfig.maximumShadowDepth) + put("viewportExpansionPx", if (options.includeOffscreenContent) -1 else observationConfig.viewportExpansionPx) + put("includeCompactText", options.includeCompactText) + } + return when (val response = gateway.request(sessionId, documentId, RuntimeMethods.CAPTURE_OBSERVATION, payload)) { + is GatewayResult.Response -> decodeObservation(response, options) + else -> gatewayFailure(response, "observation.capture") + } + } + + override suspend fun execute(command: BrowserCommand): BrowserResult = + diagnosticOperation("action.${command::class.simpleName?.lowercase() ?: "unknown"}") { + interactionMutex.withLock { executeLocked(command) } + } + + private suspend fun executeLocked(command: BrowserCommand): BrowserResult { + val documentId = activeDocumentId + ?: return failure(BrowserError.PageNotReady(_state.value.phase)) + if (closed.get()) return failure(BrowserError.SessionClosed()) + if (_state.value.phase !in OBSERVABLE_PHASES) { + return failure(BrowserError.PageNotReady(_state.value.phase)) + } + val revisionBefore = _state.value.revision + + if (command is BrowserCommand.Click && + command.button == PointerButton.PRIMARY && + configuration.actions.pointerStrategy == PointerActionStrategy.NATIVE_PREFERRED + ) { + return executeNativeClick(documentId, command) + } + + val payload = buildJsonObject { + put("command", json.encodeToJsonElement(BrowserCommand.serializer(), command)) + put("options", actionOptionsPayload()) + } + val actionTimeoutMs = configuration.actions.preparationTimeoutMs + + configuration.actions.verificationTimeoutMs + + ((command as? BrowserCommand.LongPress)?.durationMs ?: 0L) + return when (val response = gateway.request( + sessionId, + documentId, + RuntimeMethods.EXECUTE_ACTION, + payload, + timeoutMs = actionTimeoutMs, + )) { + is GatewayResult.Response -> decodeCommandReceipt(command, response) + else -> if (activeDocumentId != documentId) { + BrowserResult.Success( + CommandReceipt( + commandType = command.stableTypeName(), + strategy = command.defaultExecutionStrategy(), + documentId = documentId, + revisionBefore = revisionBefore, + revisionAfter = revisionBefore, + dispatched = true, + verified = true, + pageChanged = true, + ), + ) + } else { + gatewayFailure(response, "action.execute") + } + } + } + + private suspend fun executeNativeClick( + documentId: DocumentId, + command: BrowserCommand.Click, + ): BrowserResult { + val preparationPayload = buildJsonObject { + put("target", json.encodeToJsonElement(command.target)) + put("options", actionOptionsPayload()) + } + val preparation = when (val response = gateway.request( + sessionId, + documentId, + RuntimeMethods.PREPARE_NATIVE_CLICK, + preparationPayload, + )) { + is GatewayResult.Response -> { + if (response.envelope.status == RuntimeResponseStatus.ERROR) { + return actionRuntimeFailure(command, response.envelope.error) + } + try { + json.decodeFromJsonElement(response.envelope.result) + } catch (error: Exception) { + return failure(BrowserError.MalformedRuntimeResponse(error.message ?: "Native click preparation is malformed")) + } + } + else -> return gatewayFailure(response, "action.prepare_native_click") + } + if (preparation.viewportWidthCssPx <= 0.0 || preparation.viewportHeightCssPx <= 0.0) { + return failure(BrowserError.MalformedRuntimeResponse("Native click preparation has invalid viewport geometry")) + } + + withContext(Dispatchers.Main.immediate) { + val x = (preparation.xCssPx / preparation.viewportWidthCssPx * webView.width) + .toFloat() + .coerceIn(0f, (webView.width - 1).coerceAtLeast(0).toFloat()) + val y = (preparation.yCssPx / preparation.viewportHeightCssPx * webView.height) + .toFloat() + .coerceIn(0f, (webView.height - 1).coerceAtLeast(0).toFloat()) + val downTime = SystemClock.uptimeMillis() + val down = MotionEvent.obtain(downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0) + val up = MotionEvent.obtain(downTime, SystemClock.uptimeMillis(), MotionEvent.ACTION_UP, x, y, 0) + try { + webView.dispatchTouchEvent(down) + webView.dispatchTouchEvent(up) + } finally { + down.recycle() + up.recycle() + } + } + + val verificationPayload = buildJsonObject { put("token", preparation.token) } + return when (val response = gateway.request( + sessionId, + documentId, + RuntimeMethods.VERIFY_NATIVE_CLICK, + verificationPayload, + )) { + is GatewayResult.Response -> decodeCommandReceipt(command, response) + else -> if (activeDocumentId != documentId) { + BrowserResult.Success( + CommandReceipt( + commandType = "click", + strategy = dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.ANDROID_NATIVE_POINTER, + documentId = documentId, + revisionBefore = preparation.revisionBefore, + revisionAfter = preparation.revisionBefore, + dispatched = true, + verified = true, + pageChanged = true, + ), + ) + } else { + gatewayFailure(response, "action.verify_native_click") + } + } + } + + private fun actionOptionsPayload() = buildJsonObject { + put("geometryStableCycles", configuration.actions.geometryStableCycles) + put("geometryTolerancePx", configuration.actions.geometryTolerancePx) + } + + override suspend fun await(condition: WaitCondition): BrowserResult = + diagnosticOperation("wait.${condition::class.simpleName?.lowercase() ?: "condition"}") { + awaitInternal(condition) + } + + private suspend fun awaitInternal(condition: WaitCondition): BrowserResult { + val documentId = activeDocumentId + ?: return failure(BrowserError.PageNotReady(_state.value.phase)) + if (closed.get()) return failure(BrowserError.SessionClosed()) + + val satisfied = when (condition) { + WaitCondition.PageReady -> withTimeoutOrNull(configuration.runtime.requestTimeoutMs) { + state.first { it.phase == BrowserSessionPhase.READY } + true + } ?: false + is WaitCondition.DomQuiet -> awaitDomQuiet(condition.quietWindowMs) + is WaitCondition.ElementPresent -> { + withTimeoutOrNull(configuration.runtime.requestTimeoutMs) { + while (activeDocumentId == documentId && !closed.get()) { + when (val observation = observe(ObservationOptions(includeCompactText = false))) { + is BrowserResult.Success -> if ( + observation.value.nodes.any { it.elementRef?.let { ref -> + ref.documentId == condition.target.documentId && + ref.frameId == condition.target.frameId && + ref.elementId == condition.target.elementId + } == true } + ) return@withTimeoutOrNull true + is BrowserResult.Failure -> Unit + } + delay(ELEMENT_POLL_INTERVAL_MS) + } + false + } ?: false + } + } + return if (satisfied) { + BrowserResult.Success( + WaitReceipt( + conditionType = condition::class.simpleName ?: "condition", + satisfiedAtEpochMs = System.currentTimeMillis(), + documentId = documentId, + revision = _state.value.revision, + ), + ) + } else { + failure(BrowserError.Timeout("wait condition", configuration.runtime.requestTimeoutMs)) + } + } + + override fun close() { + if (!closed.compareAndSet(false, true)) return + transition(_state.value.copy(phase = BrowserSessionPhase.CLOSED)) + scope.launch { + try { + activeDocumentId?.let { gateway.cancelDocument(it, "Session closed") } + gateway.close("Session closed") + webView.removeJavascriptInterface(PROTOCOL_BRIDGE_NAME) + screenshotProvider.close() + documentStartScriptHandler?.remove() + documentStartScriptHandler = null + webView.stopLoading() + if (ownsWebView) { + webView.webViewClient = WebViewClient() + webView.webChromeClient = WebChromeClient() + } else { + webView.webViewClient = previousWebViewClient + webView.webChromeClient = previousWebChromeClient + } + webView.setDownloadListener(null) + if (ownsWebView) webView.destroy() + } finally { + scope.cancel() + } + } + } + + private fun configureWebView() { + webView.settings.apply { + javaScriptEnabled = true + domStorageEnabled = true + allowFileAccess = false + allowContentAccess = false + mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW + safeBrowsingEnabled = true + setSupportMultipleWindows(true) + } + } + + private fun installDocumentStartRuntime() { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.DOCUMENT_START_SCRIPT)) return + val script = loadRuntimeScript() ?: return + try { + documentStartScriptHandler = WebViewCompat.addDocumentStartJavaScript( + webView, + runtimeInstallationSource(script), + setOf("*"), + ) + } catch (error: Exception) { + emitDiagnostic( + "Document-start runtime installation failed", + mapOf("reason" to (error.message ?: error::class.java.simpleName)), + ) + } + } + + private fun installClients() { + webView.webViewClient = object : WebViewClient() { + override fun shouldOverrideUrlLoading(view: WebView, request: WebResourceRequest): Boolean { + if (closed.get() || navigationAborted) return true + return when (val decision = NavigationPolicyEvaluator.evaluate(request.url.toString(), configuration.navigation)) { + NavigationDecision.Allow -> false + is NavigationDecision.Block -> { + val blockedUrl = request.url.toString() + if (request.isForMainFrame) { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + emitDiagnostic("Blocked navigation", mapOf("reason" to decision.reason)) + } + _events.tryEmit(BrowserEvent.NavigationBlocked(blockedUrl, decision.reason)) + safelyDelegate { onNavigationBlocked(blockedUrl, decision.reason) } + true + } + } + } + + override fun onPageStarted(view: WebView, url: String?, favicon: Bitmap?) { + if (closed.get() || navigationAborted) { + view.stopLoading() + return + } + val previousDocument = activeDocumentId + val documentId = DocumentId(UUID.randomUUID().toString()) + activeDocumentId = documentId + lastHttpStatus = null + transition( + BrowserSessionState( + phase = BrowserSessionPhase.DOCUMENT_CREATED, + documentId = documentId, + revision = ObservationRevision(0), + url = url, + ), + ) + previousDocument?.let { old -> + scope.launch { gateway.cancelDocument(old, "Main document navigation started") } + } + } + + override fun onPageFinished(view: WebView, url: String?) { + if (closed.get() || navigationAborted) return + val documentId = activeDocumentId ?: return + if (_state.value.documentId != documentId || _state.value.phase != BrowserSessionPhase.DOCUMENT_CREATED) { + return + } + if (url != null && _state.value.url != null && url != _state.value.url) return + scope.launch { initializeRuntime(documentId, url) } + } + + override fun onReceivedError(view: WebView, request: WebResourceRequest, error: WebResourceError) { + if (request.isForMainFrame) { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + } + } + + override fun onReceivedHttpError(view: WebView, request: WebResourceRequest, response: WebResourceResponse) { + if (request.isForMainFrame) { + lastHttpStatus = response.statusCode + if (response.statusCode >= 400) { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + } + } + } + + override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean { + if (closed.get()) return true + lastRendererDidCrash = detail.didCrash() + transition(_state.value.copy(phase = BrowserSessionPhase.RENDERER_TERMINATED)) + _events.tryEmit(BrowserEvent.RendererTerminated(lastRendererDidCrash)) + activeDocumentId?.let { documentId -> + scope.launch { gateway.cancelDocument(documentId, "Renderer terminated") } + } + return true + } + + override fun onReceivedSslError(view: WebView, handler: SslErrorHandler, error: SslError) { + handler.cancel() + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + val url = error.url.orEmpty() + _events.tryEmit(BrowserEvent.SslErrorReceived(url, error.primaryError)) + safelyDelegate { onSslError(url, error.primaryError) } + emitDiagnostic("SSL error blocked", mapOf("url" to url, "primaryError" to error.primaryError.toString())) + } + + override fun onSafeBrowsingHit( + view: WebView, + request: WebResourceRequest, + threatType: Int, + callback: SafeBrowsingResponse, + ) { + callback.backToSafety(true) + if (request.isForMainFrame) transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + val url = request.url.toString() + _events.tryEmit(BrowserEvent.SafeBrowsingHit(url, threatType)) + safelyDelegate { onSafeBrowsingHit(url, threatType) } + emitDiagnostic("Safe Browsing threat blocked", mapOf("url" to url, "threatType" to threatType.toString())) + } + } + + webView.webChromeClient = object : WebChromeClient() { + override fun onJsAlert(view: WebView, url: String, message: String, result: JsResult): Boolean = + handleDialog(DialogType.ALERT, message, null, result) + + override fun onJsConfirm(view: WebView, url: String, message: String, result: JsResult): Boolean = + handleDialog(DialogType.CONFIRM, message, null, result) + + override fun onJsPrompt( + view: WebView, + url: String, + message: String, + defaultValue: String?, + result: JsPromptResult, + ): Boolean = handleDialog(DialogType.PROMPT, message, defaultValue, result) + + override fun onJsBeforeUnload( + view: WebView, + url: String, + message: String, + result: JsResult, + ): Boolean = handleDialog(DialogType.BEFORE_UNLOAD, message, null, result) + + override fun onCreateWindow( + view: WebView, + isDialog: Boolean, + isUserGesture: Boolean, + resultMsg: Message, + ): Boolean { + if (closed.get()) return false + val urlHint = view.hitTestResult?.extra + _events.tryEmit(BrowserEvent.PopupRequested(urlHint, isUserGesture)) + val decision = when (configuration.navigation.popupPolicy) { + PrivilegedRequestPolicy.ALLOW -> BrowserPopupDecision.OPEN_IN_CURRENT_SESSION + PrivilegedRequestPolicy.DENY -> BrowserPopupDecision.DENY + PrivilegedRequestPolicy.ASK -> safelyDelegate(BrowserPopupDecision.DENY) { + onPopup(BrowserPopupRequest(urlHint, isUserGesture)) + } + } + return decision == BrowserPopupDecision.OPEN_IN_CURRENT_SESSION && + openPopupInCurrentSession(view, resultMsg) + } + + override fun onPermissionRequest(request: PermissionRequest) { + if (closed.get()) { + request.deny() + return + } + val resources = request.resources.orEmpty().toSet() + val origin = request.origin?.toString().orEmpty() + _events.tryEmit(BrowserEvent.PermissionRequested(origin, resources.sorted())) + val approved = when (configuration.navigation.permissionPolicy) { + PrivilegedRequestPolicy.ALLOW -> resources + PrivilegedRequestPolicy.DENY -> emptySet() + PrivilegedRequestPolicy.ASK -> safelyDelegate(emptySet()) { + onPermission(BrowserPermissionRequest(origin, resources)) + } + }.intersect(resources) + if (approved.isEmpty()) request.deny() else request.grant(approved.toTypedArray()) + } + + override fun onGeolocationPermissionsShowPrompt( + origin: String, + callback: GeolocationPermissions.Callback, + ) { + if (closed.get()) { + callback.invoke(origin, false, false) + return + } + val resources = setOf(GEOLOCATION_RESOURCE) + _events.tryEmit(BrowserEvent.PermissionRequested(origin, resources.toList())) + val approved = when (configuration.navigation.permissionPolicy) { + PrivilegedRequestPolicy.ALLOW -> resources + PrivilegedRequestPolicy.DENY -> emptySet() + PrivilegedRequestPolicy.ASK -> safelyDelegate(emptySet()) { + onPermission(BrowserPermissionRequest(origin, resources)) + } + }.contains(GEOLOCATION_RESOURCE) + callback.invoke(origin, approved, false) + } + + override fun onShowFileChooser( + webView: WebView, + filePathCallback: ValueCallback>, + fileChooserParams: FileChooserParams, + ): Boolean { + if (closed.get()) { + filePathCallback.onReceiveValue(null) + return true + } + val request = BrowserFileChooserRequest( + acceptTypes = fileChooserParams.acceptTypes.orEmpty().filter { it.isNotBlank() }, + captureEnabled = fileChooserParams.isCaptureEnabled, + allowsMultiple = fileChooserParams.mode == FileChooserParams.MODE_OPEN_MULTIPLE, + ) + if (configuration.navigation.fileChooserPolicy == PrivilegedRequestPolicy.ASK) { + _events.tryEmit(BrowserEvent.FileChooserRequested(request.acceptTypes, request.captureEnabled)) + } + if (configuration.navigation.fileChooserPolicy == PrivilegedRequestPolicy.DENY) { + filePathCallback.onReceiveValue(null) + return true + } + val handled = safelyDelegate(false) { + onFileChooser(request, filePathCallback::onReceiveValue) + } + if (!handled) filePathCallback.onReceiveValue(null) + return true + } + } + + webView.setDownloadListener { url, userAgent, contentDisposition, mimeType, contentLength -> + if (closed.get()) return@setDownloadListener + val request = BrowserDownloadRequest(url, userAgent, contentDisposition, mimeType, contentLength) + _events.tryEmit(BrowserEvent.DownloadRequested(url, mimeType, contentLength.takeIf { it >= 0 })) + when (configuration.navigation.downloadPolicy) { + PrivilegedRequestPolicy.ALLOW -> safelyDelegate { onDownload(request) } + PrivilegedRequestPolicy.DENY -> Unit + PrivilegedRequestPolicy.ASK -> safelyDelegate { onDownload(request) } + } + } + } + + private fun openPopupInCurrentSession(source: WebView, resultMessage: Message): Boolean { + val transport = resultMessage.obj as? WebView.WebViewTransport ?: return false + val relay = WebView(source.context) + val relayClosed = AtomicBoolean(false) + val closeRelay = { + if (relayClosed.compareAndSet(false, true)) { + relay.stopLoading() + relay.destroy() + } + } + relay.settings.apply { + javaScriptEnabled = true + domStorageEnabled = false + allowFileAccess = false + allowContentAccess = false + mixedContentMode = WebSettings.MIXED_CONTENT_NEVER_ALLOW + safeBrowsingEnabled = true + setSupportMultipleWindows(false) + } + relay.webViewClient = object : WebViewClient() { + override fun shouldOverrideUrlLoading(view: WebView, request: WebResourceRequest): Boolean { + val url = request.url.toString() + when (val decision = NavigationPolicyEvaluator.evaluate(url, configuration.navigation)) { + NavigationDecision.Allow -> source.loadUrl(url) + is NavigationDecision.Block -> { + _events.tryEmit(BrowserEvent.NavigationBlocked(url, decision.reason)) + safelyDelegate { onNavigationBlocked(url, decision.reason) } + } + } + closeRelay() + return true + } + + override fun onReceivedError(view: WebView, request: WebResourceRequest, error: WebResourceError) { + if (request.isForMainFrame) closeRelay() + } + } + return try { + transport.webView = relay + resultMessage.sendToTarget() + scope.launch { + delay(POPUP_RELAY_TIMEOUT_MS) + closeRelay() + } + true + } catch (error: Exception) { + closeRelay() + emitDiagnostic( + "Popup relay failed", + mapOf("reason" to (error.message ?: error::class.java.simpleName)), + ) + false + } + } + + private suspend fun initializeRuntime(documentId: DocumentId, url: String?) { + if (closed.get() || activeDocumentId != documentId) return + transition(_state.value.copy(phase = BrowserSessionPhase.RUNTIME_INITIALIZING, url = url)) + val injectionFailure = injectRuntime() + if (injectionFailure != null) { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + emitDiagnostic("Runtime injection failed", mapOf("reason" to injectionFailure)) + return + } + + val configurePayload = buildJsonObject { + put("runtime", runtimePayload()) + put("experimental", experimentalPayload()) + } + val configureResult = gateway.request( + sessionId, + documentId, + RuntimeMethods.CONFIGURE, + configurePayload, + ) + if (configureResult !is GatewayResult.Response || + configureResult.envelope.status != RuntimeResponseStatus.SUCCESS + ) { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + emitDiagnostic("Runtime configuration failed", mapOf("result" to configureResult.safeDiagnosticSummary())) + return + } + + when (val ping = gateway.request(sessionId, documentId, RuntimeMethods.PING)) { + is GatewayResult.Response -> { + if (ping.envelope.status == RuntimeResponseStatus.SUCCESS && activeDocumentId == documentId) { + transition(_state.value.copy(phase = BrowserSessionPhase.INTERACTIVE, url = url)) + transition(_state.value.copy(phase = BrowserSessionPhase.STABILIZING, url = url)) + delay(configuration.runtime.readyQuietWindowMs) + if (activeDocumentId == documentId && !closed.get()) { + transition(_state.value.copy(phase = BrowserSessionPhase.READY, url = url)) + } + } else { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + } + } + else -> { + transition(_state.value.copy(phase = BrowserSessionPhase.FAILED)) + emitDiagnostic("Runtime handshake failed", mapOf("result" to ping.safeDiagnosticSummary())) + } + } + } + + private suspend fun injectRuntime(): String? { + val script = runtimeScript ?: withContext(Dispatchers.IO) { loadRuntimeScript() } + ?: return "Runtime asset '$RUNTIME_ASSET_NAME' is unavailable" + val installation = runtimeInstallationSource(script) + val existingRuntimeCheck = if (documentStartScriptHandler != null) { + """ + if (window.__AgenticWebRuntime && + window.__AgenticWebRuntime.protocolVersion === 1 && + typeof window.__AgenticWebRuntime.dispatchProtocol === 'function') { + return 'READY'; + } + """.trimIndent() + } else { + """ + if (window.__AgenticWebRuntime) { + return 'ERROR:Page defined the reserved Agentic runtime global before installation'; + } + """.trimIndent() + } + + val wrapped = """ + (function() { + try { + $existingRuntimeCheck + $installation + return window.__AgenticWebRuntime && typeof window.__AgenticWebRuntime.dispatchProtocol === 'function' + ? 'READY' + : 'MISSING_PROTOCOL'; + } catch (error) { + return 'ERROR:' + String(error && error.message ? error.message : error); + } + })(); + """.trimIndent() + + val result = withContext(Dispatchers.Main.immediate) { + suspendCancellableCoroutine { continuation -> + webView.evaluateJavascript(wrapped) { raw -> + if (!continuation.isActive) return@evaluateJavascript + continuation.resume(decodeJavascriptString(raw)) + } + } + } + return if (result == "READY") null else result ?: "Runtime injection returned no result" + } + + private fun loadRuntimeScript(): String? { + runtimeScript?.let { return it } + return try { + webView.context.assets.open(RUNTIME_ASSET_NAME).bufferedReader().use { it.readText() } + .also { runtimeScript = it } + } catch (_: Exception) { + null + } + } + + private fun runtimeInstallationSource(script: String): String = + "window.__AgenticWebRuntimeBootstrapConfiguration = " + + buildJsonObject { + put("runtime", runtimePayload()) + put("experimental", experimentalPayload()) + }.toString() + + ";\n" + script + + private fun runtimePayload() = buildJsonObject { + put("maximumMessageBytes", configuration.runtime.maximumMessageBytes) + } + + private fun experimentalPayload() = buildJsonObject { + val experimental = configuration.experimental + put("hideWebDriverProperty", experimental.hideWebDriverProperty) + put("installChromeLikeGlobals", experimental.installChromeLikeGlobals) + put("forceFutureShadowRootsOpen", experimental.forceFutureShadowRootsOpen) + } + + private suspend fun decodeObservation( + response: GatewayResult.Response, + options: ObservationOptions, + ): BrowserResult { + val envelope = response.envelope + if (envelope.status == RuntimeResponseStatus.ERROR) return runtimeFailure(envelope.error) + return try { + var observation = json.decodeFromJsonElement(envelope.result) + if (options.includeScreenshot) { + observation = when (val screenshot = screenshotProvider.capture()) { + is BrowserResult.Success -> observation.copy(screenshot = screenshot.value) + is BrowserResult.Failure -> observation.copy( + warnings = observation.warnings + ObservationWarning( + code = "SCREENSHOT_FAILED", + message = screenshot.error.message, + ), + ) + } + } + if (observation.document.id != activeDocumentId) { + failure(BrowserError.MalformedRuntimeResponse("Observation belongs to an inactive document")) + } else { + updateRevision(observation.document.id, observation.revision, observation.document.url) + BrowserResult.Success(observation) + } + } catch (error: SerializationException) { + failure(BrowserError.MalformedRuntimeResponse(error.message ?: "Observation response is malformed")) + } catch (error: IllegalArgumentException) { + failure(BrowserError.MalformedRuntimeResponse(error.message ?: "Observation response is invalid")) + } + } + + private fun decodeCommandReceipt( + command: BrowserCommand, + response: GatewayResult.Response, + ): BrowserResult { + val envelope = response.envelope + if (envelope.status == RuntimeResponseStatus.ERROR) return actionRuntimeFailure(command, envelope.error) + return try { + val receipt = json.decodeFromJsonElement(envelope.result) + if (receipt.documentId != activeDocumentId) { + return failure(BrowserError.MalformedRuntimeResponse("Action receipt belongs to an inactive document")) + } + if (receipt.commandType != command.stableTypeName()) { + return failure(BrowserError.MalformedRuntimeResponse("Action receipt command type does not match the request")) + } + updateRevision(receipt.documentId, receipt.revisionAfter) + if (!receipt.dispatched) { + failure(BrowserError.ActionRejected(command::class.simpleName ?: "command", "Runtime did not dispatch the command")) + } else if (!receipt.verified) { + failure(BrowserError.ActionNotVerified(command::class.simpleName ?: "command", receipt)) + } else { + BrowserResult.Success(receipt) + } + } catch (error: Exception) { + failure(BrowserError.MalformedRuntimeResponse(error.message ?: "Action response is malformed")) + } + } + + private suspend fun awaitDomQuiet(quietWindowMs: Long): Boolean { + return withTimeoutOrNull(configuration.runtime.requestTimeoutMs) { + while (true) { + val before = when (val observation = observe(ObservationOptions(includeCompactText = false))) { + is BrowserResult.Success -> observation.value.revision + is BrowserResult.Failure -> return@withTimeoutOrNull false + } + delay(quietWindowMs) + val after = when (val observation = observe(ObservationOptions(includeCompactText = false))) { + is BrowserResult.Success -> observation.value.revision + is BrowserResult.Failure -> return@withTimeoutOrNull false + } + if (before == after) return@withTimeoutOrNull true + } + @Suppress("UNREACHABLE_CODE") + false + } ?: false + } + + private fun handleDialog( + type: DialogType, + message: String, + defaultValue: String?, + result: JsResult, + ): Boolean { + if (closed.get()) { + result.cancel() + return true + } + return when (configuration.navigation.dialogPolicy) { + PrivilegedRequestPolicy.ALLOW -> { + if (result is JsPromptResult) result.confirm(defaultValue ?: "") else result.confirm() + true + } + PrivilegedRequestPolicy.DENY -> { + result.cancel() + true + } + PrivilegedRequestPolicy.ASK -> { + _events.tryEmit(BrowserEvent.DialogRequested(type, message.take(2_000), defaultValue?.take(2_000))) + when (val decision = safelyDelegate(BrowserDialogDecision.Cancel) { + onDialog(BrowserDialogRequest(type, message.take(2_000), defaultValue?.take(2_000))) + }) { + is BrowserDialogDecision.Confirm -> { + if (result is JsPromptResult) { + result.confirm(decision.promptValue ?: defaultValue.orEmpty()) + } else { + result.confirm() + } + } + BrowserDialogDecision.Cancel -> result.cancel() + } + true + } + } + } + + private fun transition(next: BrowserSessionState) { + val previous = _state.value + when (val reduction = BrowserLifecycleReducer.reduce(previous, next)) { + is LifecycleReduction.Accept -> { + if (previous == reduction.state) return + _state.value = reduction.state + _events.tryEmit(BrowserEvent.StateChanged(previous, reduction.state)) + } + is LifecycleReduction.Reject -> emitDiagnostic( + "Rejected lifecycle transition", + mapOf("reason" to reduction.reason), + ) + } + } + + private fun updateRevision(documentId: DocumentId, revision: ObservationRevision, url: String? = null) { + val previousRevision = _state.value.revision + transition(_state.value.copy(revision = revision, url = url ?: _state.value.url)) + if (revision != previousRevision) { + _events.tryEmit(BrowserEvent.DocumentRevisionChanged(documentId, revision)) + } + } + + private fun emitDiagnostic( + message: String, + attributes: Map, + operationId: String? = null, + durationMs: Long? = null, + ) { + if (!configuration.diagnostics.enabled) return + val safeAttributes = attributes.mapValues { (key, value) -> + if (!configuration.diagnostics.includePageContent && SENSITIVE_DIAGNOSTIC_KEYS.any { + key.contains(it, ignoreCase = true) + }) { + "[REDACTED]" + } else { + value.take(500) + } + } + diagnosticsSink?.emit( + BrowserDiagnosticEvent( + category = message, + operationId = operationId, + documentId = activeDocumentId, + revision = _state.value.revision, + durationMs = durationMs, + attributes = safeAttributes, + ), + ) + android.util.Log.d("AgenticWebView", "$message $safeAttributes") + } + + private suspend fun diagnosticOperation(category: String, block: suspend () -> T): T { + if (!configuration.diagnostics.enabled) return block() + val operationId = UUID.randomUUID().toString() + val startedAt = android.os.SystemClock.elapsedRealtime() + return try { + block() + } finally { + emitDiagnostic( + message = category, + attributes = emptyMap(), + operationId = operationId, + durationMs = android.os.SystemClock.elapsedRealtime() - startedAt, + ) + } + } + + private fun safelyDelegate(block: BrowserHostDelegate.() -> Unit) { + try { + delegate.block() + } catch (error: Exception) { + emitDiagnostic("Host delegate failure", mapOf("reason" to (error.message ?: error::class.java.simpleName))) + } + } + + private fun safelyDelegate(fallback: T, block: BrowserHostDelegate.() -> T): T = try { + delegate.block() + } catch (error: Exception) { + emitDiagnostic("Host delegate failure", mapOf("reason" to (error.message ?: error::class.java.simpleName))) + fallback + } + + private fun runtimeFailure(error: dev.shantoislam.agenticwebview.webview.protocol.RuntimeProtocolError?): BrowserResult.Failure { + if (error == null) return failure(BrowserError.MalformedRuntimeResponse("Runtime error response has no error body")) + val details = error.details.mapValues { (_, value) -> value.toString().take(1_000) } + return failure(BrowserError.RuntimeFailure(error.code, error.message, details)) + } + + private fun actionRuntimeFailure( + command: BrowserCommand, + error: dev.shantoislam.agenticwebview.webview.protocol.RuntimeProtocolError?, + ): BrowserResult.Failure { + if (error == null) return failure(BrowserError.MalformedRuntimeResponse("Runtime error response has no error body")) + val target = command.targetOrNull() + return when (error.code) { + "STALE_DOCUMENT", "STALE_ELEMENT" -> target + ?.let { failure(BrowserError.StaleElementReference(it, error.message)) } + ?: failure(BrowserError.RuntimeFailure(error.code, error.message)) + "ELEMENT_NOT_ACTIONABLE", "OPTION_NOT_FOUND" -> target + ?.let { failure(BrowserError.ElementNotActionable(it, error.message)) } + ?: failure(BrowserError.ActionRejected(command::class.simpleName ?: "command", error.message)) + "ELEMENT_OCCLUDED" -> target + ?.let { failure(BrowserError.ElementOccluded(it, message = error.message)) } + ?: failure(BrowserError.ActionRejected(command::class.simpleName ?: "command", error.message)) + "UNSUPPORTED_FRAME" -> target + ?.let { failure(BrowserError.UnsupportedFrame(it.frameId, error.message)) } + ?: failure(BrowserError.UnsupportedAction(command::class.simpleName ?: "command", error.message)) + "UNSUPPORTED_ACTION" -> failure( + BrowserError.UnsupportedAction(command::class.simpleName ?: "command", error.message), + ) + "INVALID_ACTION" -> failure( + BrowserError.ActionRejected(command::class.simpleName ?: "command", error.message), + ) + else -> runtimeFailure(error) + } + } + + private fun BrowserCommand.targetOrNull() = when (this) { + is BrowserCommand.Click -> target + is BrowserCommand.LongPress -> target + is BrowserCommand.TypeText -> target + is BrowserCommand.SelectOption -> target + is BrowserCommand.ScrollIntoView -> target + is BrowserCommand.Scroll -> (target as? dev.shantoislam.agenticwebview.api.ScrollTarget.Element)?.target + is BrowserCommand.PressKeys -> null + } + + private fun BrowserCommand.defaultExecutionStrategy() = when (this) { + is BrowserCommand.Click, is BrowserCommand.LongPress -> + dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.DOM_POINTER + is BrowserCommand.TypeText -> dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.DOM_NATIVE_SETTER + is BrowserCommand.SelectOption -> dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.DOM_SELECT + is BrowserCommand.PressKeys -> dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.DOM_KEYBOARD + is BrowserCommand.Scroll, is BrowserCommand.ScrollIntoView -> + dev.shantoislam.agenticwebview.api.ActionExecutionStrategy.DOM_SCROLL + } + + private fun BrowserCommand.stableTypeName() = when (this) { + is BrowserCommand.Click -> "click" + is BrowserCommand.LongPress -> "long_press" + is BrowserCommand.TypeText -> "type_text" + is BrowserCommand.SelectOption -> "select_option" + is BrowserCommand.PressKeys -> "press_keys" + is BrowserCommand.Scroll -> "scroll" + is BrowserCommand.ScrollIntoView -> "scroll_into_view" + } + + private fun gatewayFailure(result: GatewayResult, operation: String): BrowserResult = when (result) { + is GatewayResult.RequestRejected -> failure(BrowserError.RuntimeFailure("REQUEST_REJECTED", result.reason)) + is GatewayResult.ResponseRejected -> failure(BrowserError.MalformedRuntimeResponse(result.reason)) + is GatewayResult.PendingLimitExceeded -> failure(BrowserError.ResourceLimitExceeded("pending runtime requests", result.limit.toLong())) + is GatewayResult.TimedOut -> failure(BrowserError.Timeout(operation, result.timeoutMs)) + is GatewayResult.Cancelled -> failure(BrowserError.Cancelled(operation, result.reason)) + is GatewayResult.DispatchFailed -> failure(BrowserError.RuntimeUnavailable(result.reason)) + GatewayResult.Closed -> failure(BrowserError.SessionClosed()) + is GatewayResult.Response -> error("Response must be handled before gatewayFailure") + } + + private fun GatewayResult.safeDiagnosticSummary(): String = when (this) { + is GatewayResult.Response -> "response:${envelope.status}:${envelope.error?.code ?: "none"}" + is GatewayResult.RequestRejected -> "request_rejected" + is GatewayResult.ResponseRejected -> "response_rejected" + is GatewayResult.PendingLimitExceeded -> "pending_limit" + is GatewayResult.TimedOut -> "timeout" + is GatewayResult.Cancelled -> "cancelled" + is GatewayResult.DispatchFailed -> "dispatch_failed" + GatewayResult.Closed -> "closed" + } + + private fun failure(error: BrowserError): BrowserResult.Failure = BrowserResult.Failure(error) + + private companion object { + const val PROTOCOL_BRIDGE_NAME = "AgenticProtocolBridge" + const val RUNTIME_ASSET_NAME = "agentic_runtime.min.js" + val OBSERVABLE_PHASES = setOf( + BrowserSessionPhase.INTERACTIVE, + BrowserSessionPhase.STABILIZING, + BrowserSessionPhase.READY, + ) + const val ELEMENT_POLL_INTERVAL_MS = 100L + const val POPUP_RELAY_TIMEOUT_MS = 5_000L + const val GEOLOCATION_RESOURCE = "android.webkit.resource.GEOLOCATION" + val SENSITIVE_DIAGNOSTIC_KEYS = setOf("url", "message", "reason", "result", "payload", "text") + + fun decodeJavascriptString(raw: String?): String? { + if (raw == null || raw == "null" || raw == "undefined") return null + return try { + Json.decodeFromString(raw) + } catch (_: Exception) { + raw.trim('"') + } + } + } +} + +@Serializable +private data class NativeClickPreparation( + val token: String, + val xCssPx: Double, + val yCssPx: Double, + val viewportWidthCssPx: Double, + val viewportHeightCssPx: Double, + val revisionBefore: ObservationRevision, +) diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserHostDelegate.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserHostDelegate.kt new file mode 100644 index 0000000..693b6ec --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserHostDelegate.kt @@ -0,0 +1,50 @@ +package dev.shantoislam.agenticwebview.webview + +import android.net.Uri + +/** Synchronous decisions for requests that require host application authority. */ +interface BrowserHostDelegate { + fun onDialog(request: BrowserDialogRequest): BrowserDialogDecision = BrowserDialogDecision.Cancel + fun onPopup(request: BrowserPopupRequest): BrowserPopupDecision = BrowserPopupDecision.Deny + fun onPermission(request: BrowserPermissionRequest): Set = emptySet() + fun onDownload(request: BrowserDownloadRequest) = Unit + fun onFileChooser(request: BrowserFileChooserRequest, respond: (Array?) -> Unit): Boolean = false + fun onNavigationBlocked(url: String, reason: String) = Unit + fun onSslError(url: String, primaryError: Int) = Unit + fun onSafeBrowsingHit(url: String, threatType: Int) = Unit + + companion object { + val DenyAll: BrowserHostDelegate = object : BrowserHostDelegate {} + } +} + +data class BrowserDialogRequest( + val type: dev.shantoislam.agenticwebview.api.DialogType, + val message: String, + val defaultValue: String?, +) + +sealed interface BrowserDialogDecision { + data class Confirm(val promptValue: String? = null) : BrowserDialogDecision + data object Cancel : BrowserDialogDecision +} + +data class BrowserPopupRequest(val urlHint: String?, val isUserGesture: Boolean) + +enum class BrowserPopupDecision { OPEN_IN_CURRENT_SESSION, DENY } + +data class BrowserPermissionRequest(val origin: String, val resources: Set) + +data class BrowserDownloadRequest( + val url: String, + val userAgent: String?, + val contentDisposition: String?, + val mimeType: String?, + val contentLength: Long, +) + +data class BrowserFileChooserRequest( + val acceptTypes: List, + val captureEnabled: Boolean, + val allowsMultiple: Boolean, +) diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducer.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducer.kt new file mode 100644 index 0000000..24b8ae6 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducer.kt @@ -0,0 +1,87 @@ +package dev.shantoislam.agenticwebview.webview + +import dev.shantoislam.agenticwebview.api.BrowserSessionPhase +import dev.shantoislam.agenticwebview.api.BrowserSessionState + +internal object BrowserLifecycleReducer { + fun reduce(current: BrowserSessionState, proposed: BrowserSessionState): LifecycleReduction { + if (current.phase == proposed.phase) return LifecycleReduction.Accept(proposed) + if (proposed.phase in ALLOWED_TRANSITIONS.getValue(current.phase)) { + return LifecycleReduction.Accept(proposed) + } + return LifecycleReduction.Reject( + "Invalid browser lifecycle transition: ${current.phase} -> ${proposed.phase}", + ) + } + + private val ALLOWED_TRANSITIONS = mapOf( + BrowserSessionPhase.DETACHED to setOf( + BrowserSessionPhase.ATTACHED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.ATTACHED to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.NAVIGATING to setOf( + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.DOCUMENT_CREATED to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.RUNTIME_INITIALIZING, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.RUNTIME_INITIALIZING to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.INTERACTIVE, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.INTERACTIVE to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.STABILIZING, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.STABILIZING to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.READY, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.READY to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.FAILED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.FAILED to setOf( + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.RENDERER_TERMINATED, + BrowserSessionPhase.CLOSED, + ), + BrowserSessionPhase.RENDERER_TERMINATED to setOf(BrowserSessionPhase.CLOSED), + BrowserSessionPhase.CLOSED to emptySet(), + ) +} + +internal sealed interface LifecycleReduction { + data class Accept(val state: BrowserSessionState) : LifecycleReduction + data class Reject(val reason: String) : LifecycleReduction +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluator.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluator.kt new file mode 100644 index 0000000..842b5bb --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluator.kt @@ -0,0 +1,38 @@ +package dev.shantoislam.agenticwebview.webview + +import dev.shantoislam.agenticwebview.api.NavigationPolicy +import java.net.URI + +internal object NavigationPolicyEvaluator { + fun evaluate(url: String, policy: NavigationPolicy): NavigationDecision { + val uri = try { + URI(url) + } catch (error: Exception) { + return NavigationDecision.Block("Malformed URL: ${error.message ?: "invalid syntax"}") + } + + val scheme = uri.scheme?.lowercase() + ?: return NavigationDecision.Block("URL has no scheme") + if (scheme !in policy.allowedSchemes) { + return NavigationDecision.Block("Scheme '$scheme' is not allowed") + } + if (uri.rawUserInfo != null) { + return NavigationDecision.Block("URLs containing embedded credentials are not allowed") + } + + val host = uri.host + ?: return NavigationDecision.Block("URL has no valid host") + if (policy.deniedHosts.any { it.matches(host) }) { + return NavigationDecision.Block("Host '$host' is denied") + } + if (policy.allowedHosts.isNotEmpty() && policy.allowedHosts.none { it.matches(host) }) { + return NavigationDecision.Block("Host '$host' is not allowed") + } + return NavigationDecision.Allow + } +} + +internal sealed interface NavigationDecision { + data object Allow : NavigationDecision + data class Block(val reason: String) : NavigationDecision +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/PixelCopyScreenshotProvider.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/PixelCopyScreenshotProvider.kt new file mode 100644 index 0000000..71d7821 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/PixelCopyScreenshotProvider.kt @@ -0,0 +1,268 @@ +package dev.shantoislam.agenticwebview.webview + +import android.app.Activity +import android.content.Context +import android.content.ContextWrapper +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Rect +import android.os.Handler +import android.os.HandlerThread +import android.view.PixelCopy +import android.webkit.WebView +import dev.shantoislam.agenticwebview.api.BrowserScreenshot +import dev.shantoislam.agenticwebview.api.BrowserError +import dev.shantoislam.agenticwebview.api.BrowserResult +import dev.shantoislam.agenticwebview.api.BrowserScreenshotProvider +import dev.shantoislam.agenticwebview.api.ScreenshotConfiguration +import java.io.ByteArrayOutputStream +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject +import org.json.JSONTokener + +internal class PixelCopyScreenshotProvider( + private val webViewProvider: () -> WebView?, + private val configuration: ScreenshotConfiguration, +) : BrowserScreenshotProvider { + private val closed = AtomicBoolean(false) + private val workerThread = HandlerThread("AgenticWebViewScreenshot").apply { start() } + private val workerHandler = Handler(workerThread.looper) + + override suspend fun capture(): BrowserResult { + if (closed.get()) return failure("Screenshot provider is closed") + + val source = withContext(Dispatchers.Main.immediate) { + val webView = webViewProvider() + ?: return@withContext null + if (!webView.isAttachedToWindow || webView.width <= 0 || webView.height <= 0) { + return@withContext null + } + val activity = webView.context.findActivity() + ?: return@withContext null + val location = IntArray(2) + webView.getLocationInWindow(location) + val rect = Rect( + location[0], + location[1], + location[0] + webView.width, + location[1] + webView.height, + ) + val windowBounds = Rect(0, 0, activity.window.decorView.width, activity.window.decorView.height) + if (!rect.intersect(windowBounds) || rect.width() <= 0 || rect.height() <= 0) { + return@withContext null + } + ScreenshotSource( + activity = activity, + rect = rect, + cropOffsetX = rect.left - location[0], + cropOffsetY = rect.top - location[1], + viewWidth = webView.width, + viewHeight = webView.height, + masks = captureMasks(webView), + ) + } ?: return failure("WebView is not attached to an Activity window with positive dimensions") + + val bitmap = try { + Bitmap.createBitmap(source.rect.width(), source.rect.height(), Bitmap.Config.ARGB_8888) + } catch (error: Exception) { + return failure("Unable to allocate screenshot bitmap: ${error.message}") + } + + val copyResult = suspendCancellableCoroutine { continuation -> + try { + PixelCopy.request(source.activity.window, source.rect, bitmap, { result -> + if (continuation.isActive) continuation.resume(result) + else bitmap.recycle() + }, workerHandler) + } catch (error: Exception) { + bitmap.recycle() + if (continuation.isActive) continuation.resume(PIXEL_COPY_DISPATCH_FAILED) + } + } + if (copyResult != PixelCopy.SUCCESS) { + if (!bitmap.isRecycled) bitmap.recycle() + return failure("PixelCopy failed with code $copyResult") + } + + applyMasks(bitmap, source) + + return withContext(Dispatchers.Default) { + encode(bitmap) + } + } + + private fun encode(source: Bitmap): BrowserResult { + var outputBitmap = source + return try { + val scale = minOf( + 1f, + configuration.maximumDimensionPx.toFloat() / source.width, + configuration.maximumDimensionPx.toFloat() / source.height, + ) + if (scale < 1f) { + outputBitmap = Bitmap.createScaledBitmap( + source, + (source.width * scale).toInt().coerceAtLeast(1), + (source.height * scale).toInt().coerceAtLeast(1), + true, + ) + } + + val output = ByteArrayOutputStream() + val encoded = outputBitmap.compress(Bitmap.CompressFormat.JPEG, configuration.jpegQuality, output) + if (!encoded) return failure("Bitmap JPEG encoding failed") + val bytes = output.toByteArray() + if (bytes.size > configuration.maximumEncodedBytes) { + failure( + "Encoded screenshot is ${bytes.size} bytes; limit is ${configuration.maximumEncodedBytes}", + ) + } else { + BrowserResult.Success( + BrowserScreenshot( + bytes = bytes, + mimeType = "image/jpeg", + widthPx = outputBitmap.width, + heightPx = outputBitmap.height, + ), + ) + } + } catch (error: Exception) { + failure("Screenshot encoding failed: ${error.message}") + } finally { + if (outputBitmap !== source && !outputBitmap.isRecycled) outputBitmap.recycle() + if (!source.isRecycled) source.recycle() + } + } + + private suspend fun captureMasks(webView: WebView): ScreenshotMaskData? { + if (configuration.maskCssSelectors.isEmpty()) return null + val selectors = JSONArray(configuration.maskCssSelectors).toString() + val script = """ + (function() { + const selectors = $selectors; + const rects = []; + const visited = new WeakSet(); + function collect(root, offsetX, offsetY, depth) { + if (!root || visited.has(root) || depth > 32) return; + visited.add(root); + for (const selector of selectors) { + try { + for (const element of root.querySelectorAll(selector)) { + const r = element.getBoundingClientRect(); + if (r.width > 0 && r.height > 0) { + rects.push([offsetX + r.left, offsetY + r.top, r.width, r.height]); + } + } + } catch (_) {} + } + for (const element of root.querySelectorAll('*')) { + if (element.shadowRoot) collect(element.shadowRoot, offsetX, offsetY, depth + 1); + if (element.tagName === 'IFRAME') { + try { + const r = element.getBoundingClientRect(); + collect(element.contentDocument, offsetX + r.left, offsetY + r.top, depth + 1); + } catch (_) {} + } + } + } + collect(document, 0, 0, 0); + return JSON.stringify({ width: window.innerWidth, height: window.innerHeight, rects }); + })(); + """.trimIndent() + val raw = suspendCancellableCoroutine { continuation -> + webView.evaluateJavascript(script) { result -> + if (continuation.isActive) continuation.resume(result) + } + } ?: return null + return try { + val encoded = JSONTokener(raw).nextValue() as? String ?: return null + val root = JSONObject(encoded) + val viewportWidth = root.optDouble("width", 0.0) + val viewportHeight = root.optDouble("height", 0.0) + if (viewportWidth <= 0.0 || viewportHeight <= 0.0) return null + val entries = root.optJSONArray("rects") ?: JSONArray() + val rects = buildList { + for (index in 0 until entries.length()) { + val value = entries.optJSONArray(index) ?: continue + if (value.length() != 4) continue + add( + CssMaskRect( + value.optDouble(0), + value.optDouble(1), + value.optDouble(2), + value.optDouble(3), + ), + ) + } + } + ScreenshotMaskData(viewportWidth, viewportHeight, rects) + } catch (_: Exception) { + null + } + } + + private fun applyMasks(bitmap: Bitmap, source: ScreenshotSource) { + val masks = source.masks ?: return + val scaleX = source.viewWidth / masks.viewportWidth + val scaleY = source.viewHeight / masks.viewportHeight + val canvas = Canvas(bitmap) + val paint = Paint().apply { color = Color.BLACK } + for (mask in masks.rects) { + val left = (mask.left * scaleX - source.cropOffsetX).toFloat() + val top = (mask.top * scaleY - source.cropOffsetY).toFloat() + val right = (left + mask.width * scaleX).toFloat() + val bottom = (top + mask.height * scaleY).toFloat() + canvas.drawRect(left, top, right, bottom, paint) + } + } + + override fun close() { + if (!closed.compareAndSet(false, true)) return + workerThread.quitSafely() + } + + private data class ScreenshotSource( + val activity: Activity, + val rect: Rect, + val cropOffsetX: Int, + val cropOffsetY: Int, + val viewWidth: Int, + val viewHeight: Int, + val masks: ScreenshotMaskData?, + ) + + private data class ScreenshotMaskData( + val viewportWidth: Double, + val viewportHeight: Double, + val rects: List, + ) + + private data class CssMaskRect( + val left: Double, + val top: Double, + val width: Double, + val height: Double, + ) + + private companion object { + const val PIXEL_COPY_DISPATCH_FAILED = -1 + + fun failure(message: String): BrowserResult.Failure = + BrowserResult.Failure(BrowserError.ScreenshotFailure(message)) + } +} + +private tailrec fun Context.findActivity(): Activity? { + return when (this) { + is Activity -> this + is ContextWrapper -> if (baseContext === this) null else baseContext.findActivity() + else -> null + } +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocol.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocol.kt new file mode 100644 index 0000000..16671ed --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocol.kt @@ -0,0 +1,88 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject + +internal const val RUNTIME_PROTOCOL_VERSION: Int = 1 + +@Serializable +internal data class RuntimeRequestEnvelope( + val protocolVersion: Int = RUNTIME_PROTOCOL_VERSION, + val bridgeToken: String, + val sessionId: String, + val documentId: DocumentId, + val requestId: String, + val method: String, + val payload: JsonObject = JsonObject(emptyMap()), +) { + init { + require(sessionId.isNotBlank()) { "sessionId must not be blank" } + require(bridgeToken.length >= 32) { "bridgeToken must contain at least 32 characters" } + require(requestId.isNotBlank()) { "requestId must not be blank" } + require(method.matches(METHOD_PATTERN)) { "method has an invalid format" } + } + + private companion object { + val METHOD_PATTERN = Regex("^[a-z][a-z0-9]*(?:\\.[a-z][a-z0-9_]*)+$") + } +} + +@Serializable +internal data class RuntimeResponseEnvelope( + val protocolVersion: Int, + val bridgeToken: String, + val sessionId: String, + val documentId: DocumentId, + val requestId: String, + val status: RuntimeResponseStatus, + val result: JsonElement = JsonNull, + val error: RuntimeProtocolError? = null, +) { + fun validationError(expected: RuntimeRequestEnvelope): String? = when { + protocolVersion != RUNTIME_PROTOCOL_VERSION -> + "Protocol version mismatch: expected $RUNTIME_PROTOCOL_VERSION but received $protocolVersion" + bridgeToken != expected.bridgeToken -> "Response bridgeToken does not match request" + sessionId != expected.sessionId -> "Response sessionId does not match request" + documentId != expected.documentId -> "Response documentId does not match request" + requestId != expected.requestId -> "Response requestId does not match request" + status == RuntimeResponseStatus.SUCCESS && error != null -> "Successful response must not contain an error" + status == RuntimeResponseStatus.ERROR && error == null -> "Error response must contain an error" + else -> null + } +} + +@Serializable +internal enum class RuntimeResponseStatus { + @kotlinx.serialization.SerialName("success") + SUCCESS, + @kotlinx.serialization.SerialName("error") + ERROR, +} + +@Serializable +internal data class RuntimeProtocolError( + val code: String, + val message: String, + val details: JsonObject = JsonObject(emptyMap()), +) { + init { + require(code.matches(CODE_PATTERN)) { "Runtime error code has an invalid format" } + require(message.isNotBlank()) { "Runtime error message must not be blank" } + } + + private companion object { + val CODE_PATTERN = Regex("^[A-Z][A-Z0-9_]*$") + } +} + +internal object RuntimeMethods { + const val PING = "system.ping" + const val CONFIGURE = "runtime.configure" + const val CAPTURE_OBSERVATION = "observation.capture" + const val EXECUTE_ACTION = "action.execute" + const val PREPARE_NATIVE_CLICK = "action.prepare_native_click" + const val VERIFY_NATIVE_CLICK = "action.verify_native_click" +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolBridge.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolBridge.kt new file mode 100644 index 0000000..6cd7c22 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolBridge.kt @@ -0,0 +1,95 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import android.os.SystemClock +import android.webkit.JavascriptInterface +import java.util.concurrent.atomic.AtomicInteger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch + +internal class RuntimeProtocolBridge( + private val scope: CoroutineScope, + private val gatewayProvider: () -> RuntimeProtocolGateway?, + private val maximumMessageBytes: Int, + private val maximumConcurrentCallbacks: Int, + private val onRejectedResponse: (String) -> Unit = {}, + private val transportErrorHandler: (String) -> Unit = {}, +) { + private val activeCallbacks = AtomicInteger(0) + private val callbackRateLock = Any() + private var callbackWindowStartedAtMs = SystemClock.elapsedRealtime() + private var callbacksInWindow = 0 + private var rateLimitReported = false + + init { + require(maximumMessageBytes > 0) { "maximumMessageBytes must be positive" } + require(maximumConcurrentCallbacks > 0) { "maximumConcurrentCallbacks must be positive" } + } + + @JavascriptInterface + fun onResponse(rawResponse: String) { + if (rawResponse.length > maximumMessageBytes) { + onRejectedResponse("Runtime response exceeds the bridge message limit") + return + } + if (!acquireCallback()) return + scope.launch { + try { + when (val result = gatewayProvider()?.acceptResponse(rawResponse)) { + is IncomingResponseResult.Rejected -> onRejectedResponse(result.reason) + IncomingResponseResult.AlreadyCompleted -> onRejectedResponse("Runtime response was already completed") + IncomingResponseResult.UnknownRequest -> onRejectedResponse("Runtime response has an unknown requestId") + IncomingResponseResult.Completed -> Unit + null -> onRejectedResponse("Runtime gateway is unavailable") + } + } finally { + activeCallbacks.decrementAndGet() + } + } + } + + @JavascriptInterface + fun onTransportError(message: String) { + if (!acquireCallback()) return + val boundedMessage = message.take(MAX_ERROR_MESSAGE_LENGTH) + scope.launch { + try { + transportErrorHandler(boundedMessage) + } finally { + activeCallbacks.decrementAndGet() + } + } + } + + private fun acquireCallback(): Boolean { + val withinRateLimit = synchronized(callbackRateLock) { + val now = SystemClock.elapsedRealtime() + if (now - callbackWindowStartedAtMs >= CALLBACK_WINDOW_MS) { + callbackWindowStartedAtMs = now + callbacksInWindow = 0 + rateLimitReported = false + } + if (callbacksInWindow >= maxOf(MINIMUM_CALLBACKS_PER_WINDOW, maximumConcurrentCallbacks * 4)) { + val shouldReport = !rateLimitReported + rateLimitReported = true + shouldReport to false + } else { + callbacksInWindow++ + false to true + } + } + if (!withinRateLimit.second) { + if (withinRateLimit.first) onRejectedResponse("Runtime callback rate limit exceeded") + return false + } + if (activeCallbacks.incrementAndGet() <= maximumConcurrentCallbacks) return true + activeCallbacks.decrementAndGet() + onRejectedResponse("Runtime callback limit exceeded") + return false + } + + private companion object { + const val MAX_ERROR_MESSAGE_LENGTH = 2_000 + const val CALLBACK_WINDOW_MS = 1_000L + const val MINIMUM_CALLBACKS_PER_WINDOW = 16 + } +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodec.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodec.kt new file mode 100644 index 0000000..eb53cec --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodec.kt @@ -0,0 +1,58 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +internal class RuntimeProtocolCodec( + private val maximumMessageBytes: Int, + private val json: Json = Json { + ignoreUnknownKeys = false + explicitNulls = false + encodeDefaults = true + }, +) { + init { require(maximumMessageBytes > 0) { "maximumMessageBytes must be positive" } } + + fun encodeRequest(request: RuntimeRequestEnvelope): CodecResult = encode { + json.encodeToString(request) + } + + fun encodeResponse(response: RuntimeResponseEnvelope): CodecResult = encode { + json.encodeToString(response) + } + + fun decodeResponse(raw: String): CodecResult { + val byteCount = raw.toByteArray(Charsets.UTF_8).size + if (byteCount > maximumMessageBytes) { + return CodecResult.Failure("Runtime response is $byteCount bytes; limit is $maximumMessageBytes") + } + return try { + CodecResult.Success(json.decodeFromString(raw)) + } catch (error: SerializationException) { + CodecResult.Failure("Malformed runtime response: ${error.message}") + } catch (error: IllegalArgumentException) { + CodecResult.Failure("Invalid runtime response: ${error.message}") + } + } + + private inline fun encode(block: () -> String): CodecResult = try { + val encoded = block() + val byteCount = encoded.toByteArray(Charsets.UTF_8).size + if (byteCount > maximumMessageBytes) { + CodecResult.Failure("Encoded runtime message is $byteCount bytes; limit is $maximumMessageBytes") + } else { + CodecResult.Success(encoded) + } + } catch (error: SerializationException) { + CodecResult.Failure("Failed to encode runtime message: ${error.message}") + } catch (error: IllegalArgumentException) { + CodecResult.Failure("Invalid runtime message: ${error.message}") + } +} + +internal sealed interface CodecResult { + data class Success(val value: T) : CodecResult + data class Failure(val reason: String) : CodecResult +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGateway.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGateway.kt new file mode 100644 index 0000000..0a22688 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGateway.kt @@ -0,0 +1,94 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import java.util.UUID +import kotlinx.serialization.json.JsonObject + +internal class RuntimeProtocolGateway( + private val bridgeToken: String, + maximumPendingRequests: Int, + maximumMessageBytes: Int, + private val requestTimeoutMs: Long, + private val transport: RuntimeTransport, + private val requestIdFactory: () -> String = { UUID.randomUUID().toString() }, +) { + private val codec = RuntimeProtocolCodec(maximumMessageBytes) + private val registry = RuntimeRequestRegistry(maximumPendingRequests) + + init { require(requestTimeoutMs > 0) { "requestTimeoutMs must be positive" } } + + suspend fun request( + sessionId: String, + documentId: DocumentId, + method: String, + payload: JsonObject = JsonObject(emptyMap()), + timeoutMs: Long = requestTimeoutMs, + ): GatewayResult { + if (timeoutMs <= 0) return GatewayResult.RequestRejected("timeoutMs must be positive") + val envelope = try { + RuntimeRequestEnvelope( + bridgeToken = bridgeToken, + sessionId = sessionId, + documentId = documentId, + requestId = requestIdFactory(), + method = method, + payload = payload, + ) + } catch (error: IllegalArgumentException) { + return GatewayResult.RequestRejected(error.message ?: "Invalid runtime request") + } + + return when (val result = registry.dispatchAndAwait(envelope, timeoutMs) { request -> + when (val encoded = codec.encodeRequest(request)) { + is CodecResult.Success -> transport.dispatch(encoded.value) + is CodecResult.Failure -> throw RuntimeDispatchException(encoded.reason) + } + }) { + is RegistryResult.Response -> GatewayResult.Response(result.value) + is RegistryResult.Rejected -> GatewayResult.ResponseRejected(result.reason) + is RegistryResult.LimitExceeded -> GatewayResult.PendingLimitExceeded(result.limit) + is RegistryResult.TimedOut -> GatewayResult.TimedOut(result.timeoutMs) + is RegistryResult.Cancelled -> GatewayResult.Cancelled(result.reason) + is RegistryResult.DispatchFailed -> GatewayResult.DispatchFailed(result.reason) + RegistryResult.Closed -> GatewayResult.Closed + } + } + + suspend fun acceptResponse(raw: String): IncomingResponseResult = when (val decoded = codec.decodeResponse(raw)) { + is CodecResult.Failure -> IncomingResponseResult.Rejected(decoded.reason) + is CodecResult.Success -> when (registry.complete(decoded.value)) { + CompletionResult.Completed -> IncomingResponseResult.Completed + CompletionResult.AlreadyCompleted -> IncomingResponseResult.AlreadyCompleted + CompletionResult.UnknownRequest -> IncomingResponseResult.UnknownRequest + } + } + + suspend fun cancelDocument(documentId: DocumentId, reason: String): Int = + registry.cancelDocument(documentId, reason) + + suspend fun close(reason: String = "Runtime gateway closed"): Int = registry.close(reason) +} + +internal fun interface RuntimeTransport { + suspend fun dispatch(encodedRequest: String) +} + +internal sealed interface GatewayResult { + data class Response(val envelope: RuntimeResponseEnvelope) : GatewayResult + data class RequestRejected(val reason: String) : GatewayResult + data class ResponseRejected(val reason: String) : GatewayResult + data class PendingLimitExceeded(val limit: Int) : GatewayResult + data class TimedOut(val timeoutMs: Long) : GatewayResult + data class Cancelled(val reason: String) : GatewayResult + data class DispatchFailed(val reason: String) : GatewayResult + data object Closed : GatewayResult +} + +internal sealed interface IncomingResponseResult { + data object Completed : IncomingResponseResult + data object AlreadyCompleted : IncomingResponseResult + data object UnknownRequest : IncomingResponseResult + data class Rejected(val reason: String) : IncomingResponseResult +} + +private class RuntimeDispatchException(message: String) : RuntimeException(message) diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistry.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistry.kt new file mode 100644 index 0000000..a75510f --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistry.kt @@ -0,0 +1,115 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout + +internal class RuntimeRequestRegistry( + private val maximumPendingRequests: Int, +) { + private val mutex = Mutex() + private val pending = LinkedHashMap() + private val closed = AtomicBoolean(false) + + init { require(maximumPendingRequests > 0) { "maximumPendingRequests must be positive" } } + + suspend fun dispatchAndAwait( + request: RuntimeRequestEnvelope, + timeoutMs: Long, + dispatch: suspend (RuntimeRequestEnvelope) -> Unit, + ): RegistryResult { + require(timeoutMs > 0) { "timeoutMs must be positive" } + + val deferred = CompletableDeferred() + val registration = mutex.withLock { + when { + closed.get() -> RegistryResult.Closed + request.requestId in pending -> RegistryResult.Rejected("Duplicate requestId: ${request.requestId}") + pending.size >= maximumPendingRequests -> RegistryResult.LimitExceeded(maximumPendingRequests) + else -> { + pending[request.requestId] = PendingRequest(request, deferred) + null + } + } + } + if (registration != null) return registration + + return try { + dispatch(request) + when (val signal = withTimeout(timeoutMs) { deferred.await() }) { + is RegistrySignal.Response -> { + val validationError = signal.value.validationError(request) + if (validationError == null) RegistryResult.Response(signal.value) + else RegistryResult.Rejected(validationError) + } + is RegistrySignal.Cancelled -> RegistryResult.Cancelled(signal.reason) + } + } catch (_: TimeoutCancellationException) { + RegistryResult.TimedOut(timeoutMs) + } catch (cancellation: CancellationException) { + throw cancellation + } catch (error: Exception) { + RegistryResult.DispatchFailed(error.message ?: error::class.java.simpleName) + } finally { + withContext(NonCancellable) { + mutex.withLock { pending.remove(request.requestId) } + } + } + } + + suspend fun complete(response: RuntimeResponseEnvelope): CompletionResult { + val target = mutex.withLock { pending[response.requestId] } + ?: return CompletionResult.UnknownRequest + return if (target.deferred.complete(RegistrySignal.Response(response))) { + CompletionResult.Completed + } else { + CompletionResult.AlreadyCompleted + } + } + + suspend fun cancelDocument(documentId: DocumentId, reason: String): Int { + val targets = mutex.withLock { + pending.values.filter { it.request.documentId == documentId } + } + targets.forEach { it.deferred.complete(RegistrySignal.Cancelled(reason)) } + return targets.size + } + + suspend fun close(reason: String = "Runtime request registry closed"): Int { + if (!closed.compareAndSet(false, true)) return 0 + val targets = mutex.withLock { pending.values.toList() } + targets.forEach { it.deferred.complete(RegistrySignal.Cancelled(reason)) } + return targets.size + } + + suspend fun pendingCount(): Int = mutex.withLock { pending.size } + + private data class PendingRequest( + val request: RuntimeRequestEnvelope, + val deferred: CompletableDeferred, + ) +} + +internal sealed interface RegistryResult { + data class Response(val value: RuntimeResponseEnvelope) : RegistryResult + data class Rejected(val reason: String) : RegistryResult + data class LimitExceeded(val limit: Int) : RegistryResult + data class TimedOut(val timeoutMs: Long) : RegistryResult + data class Cancelled(val reason: String) : RegistryResult + data class DispatchFailed(val reason: String) : RegistryResult + data object Closed : RegistryResult +} + +internal enum class CompletionResult { Completed, AlreadyCompleted, UnknownRequest } + +private sealed interface RegistrySignal { + data class Response(val value: RuntimeResponseEnvelope) : RegistrySignal + data class Cancelled(val reason: String) : RegistrySignal +} diff --git a/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/WebViewRuntimeTransport.kt b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/WebViewRuntimeTransport.kt new file mode 100644 index 0000000..eef79a2 --- /dev/null +++ b/browser-webview/src/main/kotlin/dev/shantoislam/agenticwebview/webview/protocol/WebViewRuntimeTransport.kt @@ -0,0 +1,72 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import android.webkit.WebView +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import org.json.JSONTokener +import org.json.JSONObject + +internal class WebViewRuntimeTransport( + private val webViewProvider: () -> WebView?, +) : RuntimeTransport { + override suspend fun dispatch(encodedRequest: String) = withContext(Dispatchers.Main.immediate) { + val webView = webViewProvider() + ?: throw IllegalStateException("WebView is not attached") + val requestLiteral = JSONObject.quote(encodedRequest) + suspendCancellableCoroutine { continuation -> + webView.evaluateJavascript( + """ + (function() { + try { + var runtime = window.__AgenticWebRuntime; + if (!runtime || typeof runtime.dispatchProtocol !== 'function') { + return 'ERROR:Agentic runtime protocol is unavailable'; + } + runtime.dispatchProtocol($requestLiteral).then( + function(response) { + if (window.AgenticProtocolBridge) { + window.AgenticProtocolBridge.onResponse(response); + } + }, + function(error) { + if (window.AgenticProtocolBridge) { + window.AgenticProtocolBridge.onTransportError(String(error)); + } + } + ); + return 'DISPATCHED'; + } catch (error) { + return 'ERROR:' + String(error && error.message ? error.message : error); + } + })(); + """.trimIndent(), + ) { rawResult -> + if (!continuation.isActive) return@evaluateJavascript + val result = decodeJavascriptString(rawResult) + if (result == "DISPATCHED") { + continuation.resume(Unit) + } else { + continuation.resumeWithException( + IllegalStateException(result?.removePrefix("ERROR:") ?: "Runtime dispatch returned no result"), + ) + } + } + } + } + + private fun decodeJavascriptString(raw: String?): String? { + if (raw == null || raw == "null" || raw == "undefined") return null + return try { + when (val decoded = JSONTokener(raw).nextValue()) { + JSONObject.NULL -> null + is String -> decoded + else -> decoded.toString() + } + } catch (_: Exception) { + raw + } + } +} diff --git a/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducerTest.kt b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducerTest.kt new file mode 100644 index 0000000..9de88e9 --- /dev/null +++ b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/BrowserLifecycleReducerTest.kt @@ -0,0 +1,61 @@ +package dev.shantoislam.agenticwebview.webview + +import dev.shantoislam.agenticwebview.api.BrowserSessionPhase +import dev.shantoislam.agenticwebview.api.BrowserSessionState +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class BrowserLifecycleReducerTest { + @Test + fun acceptsNormalInitializationSequence() { + val phases = listOf( + BrowserSessionPhase.ATTACHED, + BrowserSessionPhase.NAVIGATING, + BrowserSessionPhase.DOCUMENT_CREATED, + BrowserSessionPhase.RUNTIME_INITIALIZING, + BrowserSessionPhase.INTERACTIVE, + BrowserSessionPhase.STABILIZING, + BrowserSessionPhase.READY, + ) + var state = BrowserSessionState(BrowserSessionPhase.DETACHED) + + phases.forEach { phase -> + val reduction = BrowserLifecycleReducer.reduce(state, state.copy(phase = phase)) + assertTrue(reduction is LifecycleReduction.Accept) + state = (reduction as LifecycleReduction.Accept).state + } + + assertEquals(BrowserSessionPhase.READY, state.phase) + } + + @Test + fun closedSessionRejectsLateCallbacks() { + val closed = BrowserSessionState(BrowserSessionPhase.CLOSED) + + val reduction = BrowserLifecycleReducer.reduce( + closed, + BrowserSessionState(BrowserSessionPhase.DOCUMENT_CREATED), + ) + + assertTrue(reduction is LifecycleReduction.Reject) + } + + @Test + fun rendererTerminationIsTerminalUntilClose() { + val terminated = BrowserSessionState(BrowserSessionPhase.RENDERER_TERMINATED) + + assertTrue( + BrowserLifecycleReducer.reduce( + terminated, + BrowserSessionState(BrowserSessionPhase.READY), + ) is LifecycleReduction.Reject, + ) + assertTrue( + BrowserLifecycleReducer.reduce( + terminated, + BrowserSessionState(BrowserSessionPhase.CLOSED), + ) is LifecycleReduction.Accept, + ) + } +} diff --git a/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluatorTest.kt b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluatorTest.kt new file mode 100644 index 0000000..594c580 --- /dev/null +++ b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/NavigationPolicyEvaluatorTest.kt @@ -0,0 +1,61 @@ +package dev.shantoislam.agenticwebview.webview + +import dev.shantoislam.agenticwebview.api.HostRule +import dev.shantoislam.agenticwebview.api.NavigationPolicy +import org.junit.Assert.assertTrue +import org.junit.Test + +class NavigationPolicyEvaluatorTest { + @Test + fun allowsConfiguredDomainAndItsSubdomains() { + val policy = NavigationPolicy( + allowedHosts = setOf(HostRule.DomainAndSubdomains("example.com")), + ) + + assertTrue(NavigationPolicyEvaluator.evaluate("https://example.com/path", policy) is NavigationDecision.Allow) + assertTrue(NavigationPolicyEvaluator.evaluate("https://app.example.com/path", policy) is NavigationDecision.Allow) + } + + @Test + fun doesNotAllowSuffixConfusion() { + val policy = NavigationPolicy( + allowedHosts = setOf(HostRule.DomainAndSubdomains("example.com")), + ) + + assertTrue(NavigationPolicyEvaluator.evaluate("https://evilexample.com", policy) is NavigationDecision.Block) + } + + @Test + fun deniedRuleTakesPrecedence() { + val policy = NavigationPolicy( + allowedHosts = setOf(HostRule.DomainAndSubdomains("example.com")), + deniedHosts = setOf(HostRule.Exact("private.example.com")), + ) + + assertTrue(NavigationPolicyEvaluator.evaluate("https://private.example.com", policy) is NavigationDecision.Block) + } + + @Test + fun rejectsExternalSchemesByDefault() { + assertTrue( + NavigationPolicyEvaluator.evaluate("intent://example.com", NavigationPolicy()) is NavigationDecision.Block, + ) + } + + @Test + fun rejectsPlainHttpByDefault() { + assertTrue( + NavigationPolicyEvaluator.evaluate("http://example.com", NavigationPolicy()) is NavigationDecision.Block, + ) + } + + @Test + fun rejectsEmbeddedUrlCredentials() { + assertTrue( + NavigationPolicyEvaluator.evaluate( + "https://user:password@example.com/private", + NavigationPolicy(), + ) is NavigationDecision.Block, + ) + } +} diff --git a/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodecTest.kt b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodecTest.kt new file mode 100644 index 0000000..a860e65 --- /dev/null +++ b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolCodecTest.kt @@ -0,0 +1,94 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.put +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class RuntimeProtocolCodecTest { + private val codec = RuntimeProtocolCodec(maximumMessageBytes = 8_192) + + @Test + fun requestRoundTripProducesExpectedEnvelope() { + val request = request(payloadValue = "hello") + val encoded = codec.encodeRequest(request) + + assertTrue(encoded is CodecResult.Success) + val value = (encoded as CodecResult.Success).value + assertTrue(value.contains("\"protocolVersion\":1")) + assertTrue(value.contains("\"method\":\"system.ping\"")) + } + + @Test + fun malformedResponseIsRejected() { + val result = codec.decodeResponse("{not-json") + + assertTrue(result is CodecResult.Failure) + } + + @Test + fun oversizedResponseIsRejectedBeforeParsing() { + val result = RuntimeProtocolCodec(maximumMessageBytes = 16).decodeResponse("x".repeat(17)) + + assertTrue(result is CodecResult.Failure) + } + + @Test + fun responseValidationRejectsWrongDocument() { + val request = request() + val response = successResponse(request).copy(documentId = DocumentId("other-document")) + + assertEquals("Response documentId does not match request", response.validationError(request)) + } + + @Test + fun responseValidationRejectsSpoofedBridgeToken() { + val request = request() + val response = successResponse(request).copy(bridgeToken = "different-token-123456789012345678901234567890") + + assertEquals("Response bridgeToken does not match request", response.validationError(request)) + } + + @Test + fun sharedGoldenFixturesDecodeWithTheKotlinProtocol() { + val success = fixture("v1/response-ping-success.json") + val failure = fixture("v1/response-runtime-error.json") + + val successResult = codec.decodeResponse(success) + val failureResult = codec.decodeResponse(failure) + + assertTrue(successResult is CodecResult.Success) + assertTrue(failureResult is CodecResult.Success) + assertEquals(RuntimeResponseStatus.SUCCESS, (successResult as CodecResult.Success).value.status) + assertEquals("NOT_READY", (failureResult as CodecResult.Success).value.error?.code) + } + + private fun fixture(name: String): String = + requireNotNull(javaClass.classLoader?.getResource(name)) { "Missing protocol fixture: $name" }.readText() + + private fun request(payloadValue: String = "value") = RuntimeRequestEnvelope( + bridgeToken = TEST_BRIDGE_TOKEN, + sessionId = "session-1", + documentId = DocumentId("document-1"), + requestId = "request-1", + method = RuntimeMethods.PING, + payload = buildJsonObject { put("value", payloadValue) }, + ) + + private fun successResponse(request: RuntimeRequestEnvelope) = RuntimeResponseEnvelope( + protocolVersion = RUNTIME_PROTOCOL_VERSION, + bridgeToken = request.bridgeToken, + sessionId = request.sessionId, + documentId = request.documentId, + requestId = request.requestId, + status = RuntimeResponseStatus.SUCCESS, + result = buildJsonObject { put("ok", true) }, + ) + + private companion object { + const val TEST_BRIDGE_TOKEN = "bridge-token-12345678901234567890123456789012" + } +} diff --git a/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGatewayTest.kt b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGatewayTest.kt new file mode 100644 index 0000000..1169555 --- /dev/null +++ b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeProtocolGatewayTest.kt @@ -0,0 +1,77 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import kotlinx.coroutines.async +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.yield +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonNull +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class RuntimeProtocolGatewayTest { + private val json = Json { explicitNulls = false; encodeDefaults = true } + + @Test + fun transportReceivesRegisteredRequestAndResponseCompletesIt() = runTest { + lateinit var gateway: RuntimeProtocolGateway + val transport = RuntimeTransport { rawRequest -> + val request = json.decodeFromString(rawRequest) + val response = RuntimeResponseEnvelope( + protocolVersion = RUNTIME_PROTOCOL_VERSION, + bridgeToken = request.bridgeToken, + sessionId = request.sessionId, + documentId = request.documentId, + requestId = request.requestId, + status = RuntimeResponseStatus.SUCCESS, + result = JsonNull, + ) + assertEquals(IncomingResponseResult.Completed, gateway.acceptResponse(json.encodeToString(response))) + } + gateway = gateway(transport) + + val result = gateway.request("session-1", DocumentId("document-1"), RuntimeMethods.PING) + + assertTrue(result is GatewayResult.Response) + } + + @Test + fun malformedIncomingResponseDoesNotCompletePendingRequest() = runTest { + var capturedRequest: RuntimeRequestEnvelope? = null + val gateway = gateway(RuntimeTransport { raw -> + capturedRequest = json.decodeFromString(raw) + }) + val result = async { + gateway.request("session-1", DocumentId("document-1"), RuntimeMethods.PING) + } + while (capturedRequest == null) yield() + + val incoming = gateway.acceptResponse("{invalid") + assertTrue(incoming is IncomingResponseResult.Rejected) + + gateway.cancelDocument(DocumentId("document-1"), "test complete") + assertEquals(GatewayResult.Cancelled("test complete"), result.await()) + } + + @Test + fun closedGatewayRejectsNewRequests() = runTest { + val gateway = gateway(RuntimeTransport { }) + gateway.close() + + val result = gateway.request("session-1", DocumentId("document-1"), RuntimeMethods.PING) + + assertEquals(GatewayResult.Closed, result) + } + + private fun gateway(transport: RuntimeTransport) = RuntimeProtocolGateway( + bridgeToken = "bridge-token-12345678901234567890123456789012", + maximumPendingRequests = 4, + maximumMessageBytes = 8_192, + requestTimeoutMs = 1_000, + transport = transport, + requestIdFactory = { "request-1" }, + ) +} diff --git a/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistryTest.kt b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistryTest.kt new file mode 100644 index 0000000..2f50f9c --- /dev/null +++ b/browser-webview/src/test/kotlin/dev/shantoislam/agenticwebview/webview/protocol/RuntimeRequestRegistryTest.kt @@ -0,0 +1,103 @@ +package dev.shantoislam.agenticwebview.webview.protocol + +import dev.shantoislam.agenticwebview.api.DocumentId +import kotlinx.coroutines.async +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.test.runTest +import kotlinx.coroutines.yield +import kotlinx.serialization.json.JsonNull +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class RuntimeRequestRegistryTest { + @Test + fun requestIsRegisteredBeforeDispatch() = runTest { + val registry = RuntimeRequestRegistry(maximumPendingRequests = 4) + val request = request("request-1") + + val result = registry.dispatchAndAwait(request, 1_000) { + assertEquals(1, registry.pendingCount()) + registry.complete(successResponse(it)) + } + + assertTrue(result is RegistryResult.Response) + assertEquals(0, registry.pendingCount()) + } + + @Test + fun duplicateResponseCompletesRequestOnlyOnce() = runTest { + val registry = RuntimeRequestRegistry(maximumPendingRequests = 4) + val request = request("request-1") + var secondCompletion: CompletionResult? = null + + val result = registry.dispatchAndAwait(request, 1_000) { + val response = successResponse(it) + assertEquals(CompletionResult.Completed, registry.complete(response)) + secondCompletion = registry.complete(response) + } + + assertTrue(result is RegistryResult.Response) + assertEquals(CompletionResult.AlreadyCompleted, secondCompletion) + } + + @Test + fun documentCancellationCompletesMatchingRequest() = runTest { + val registry = RuntimeRequestRegistry(maximumPendingRequests = 4) + val request = request("request-1") + + val result = async { + registry.dispatchAndAwait(request, 5_000) { } + } + while (registry.pendingCount() == 0) yield() + + assertEquals(1, registry.cancelDocument(request.documentId, "navigation started")) + assertEquals(RegistryResult.Cancelled("navigation started"), result.await()) + } + + @Test + fun pendingLimitRejectsAdditionalRequest() = runTest { + val registry = RuntimeRequestRegistry(maximumPendingRequests = 1) + val first = async { registry.dispatchAndAwait(request("first"), 5_000) { } } + while (registry.pendingCount() == 0) yield() + + val second = registry.dispatchAndAwait(request("second"), 1_000) { } + + assertEquals(RegistryResult.LimitExceeded(1), second) + registry.cancelDocument(DocumentId("document-1"), "test finished") + first.await() + } + + @Test + fun callerCancellationAlwaysRemovesPendingRequest() = runTest { + val registry = RuntimeRequestRegistry(maximumPendingRequests = 1) + val pending = async { registry.dispatchAndAwait(request("request-1"), 5_000) { } } + while (registry.pendingCount() == 0) yield() + + pending.cancelAndJoin() + + assertEquals(0, registry.pendingCount()) + } + + private fun request(id: String) = RuntimeRequestEnvelope( + bridgeToken = TEST_BRIDGE_TOKEN, + sessionId = "session-1", + documentId = DocumentId("document-1"), + requestId = id, + method = RuntimeMethods.PING, + ) + + private fun successResponse(request: RuntimeRequestEnvelope) = RuntimeResponseEnvelope( + protocolVersion = RUNTIME_PROTOCOL_VERSION, + bridgeToken = request.bridgeToken, + sessionId = request.sessionId, + documentId = request.documentId, + requestId = request.requestId, + status = RuntimeResponseStatus.SUCCESS, + result = JsonNull, + ) + + private companion object { + const val TEST_BRIDGE_TOKEN = "bridge-token-12345678901234567890123456789012" + } +} diff --git a/build.gradle.kts b/build.gradle.kts index c13f8a2..9742aff 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,44 @@ +import groovy.json.JsonSlurper + // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { alias(libs.plugins.android.application) apply false alias(libs.plugins.android.library) apply false + alias(libs.plugins.kotlin.android) apply false + alias(libs.plugins.kotlin.jvm) apply false alias(libs.plugins.kotlin.compose) apply false alias(libs.plugins.kotlin.serialization) apply false alias(libs.plugins.maven.publish) apply false -} \ No newline at end of file +} + +val buildWebRuntime by tasks.registering(Exec::class) { + group = "build" + description = "Builds the TypeScript runtime bundled by Android SDK modules." + workingDir = file("web-runtime") + inputs.files(fileTree("web-runtime/src"), file("web-runtime/package.json"), file("web-runtime/package-lock.json")) + outputs.file(file("web-runtime/dist/agentic_runtime.min.js")) + if (System.getProperty("os.name").lowercase().contains("windows")) { + commandLine("cmd", "/c", "npm", "run", "build") + } else { + commandLine("npm", "run", "build") + } +} + +val verifyVersionAlignment by tasks.registering { + group = "verification" + description = "Checks that Gradle and the private web runtime use the same base version." + inputs.file("gradle.properties") + inputs.file("web-runtime/package.json") + doLast { + val gradleVersion = providers.gradleProperty("VERSION_NAME").get().substringBefore('-') + val packageJson = JsonSlurper().parse(file("web-runtime/package.json")) as Map<*, *> + val runtimeVersion = packageJson["version"]?.toString() + check(runtimeVersion == gradleVersion) { + "web-runtime version $runtimeVersion does not match Gradle base version $gradleVersion" + } + } +} + +buildWebRuntime.configure { + dependsOn(verifyVersionAlignment) +} diff --git a/docs/adr/0001-no-compatibility-rewrite.md b/docs/adr/0001-no-compatibility-rewrite.md new file mode 100644 index 0000000..fc50cba --- /dev/null +++ b/docs/adr/0001-no-compatibility-rewrite.md @@ -0,0 +1,16 @@ +# ADR 0001: Replace the prototype without compatibility + +- Status: accepted +- Date: 2026-09-01 + +## Decision + +Replace the prototype controller/custom-view architecture with separate contract, WebView, Compose, generic-tool, adapter, and page-runtime modules. Delete the former API, parser, selector/hash identity, bridge, and duplicated configuration rather than maintain an adapter façade. + +## Rationale + +The prototype mixed UI ownership, browser behavior, agent formatting, and runtime transport. Compatibility would preserve two lifecycles and two error/configuration models, making cancellation, security policy, and element identity ambiguous. The project is still in development, so a clean break has lower long-term risk. + +## Consequences + +Consumers must migrate to `AgenticBrowserHost` and `AgenticBrowserSession`. Every behavior is now typed and every framework integration goes through `AgentToolDispatcher`. Protocol and artifact versioning become explicit. The repository carries no deprecated prototype path. diff --git a/docs/agent-perception.md b/docs/agent-perception.md deleted file mode 100644 index 1153e23..0000000 --- a/docs/agent-perception.md +++ /dev/null @@ -1,90 +0,0 @@ -# Agent Integration Guide - -This guide explains how to use the `AgenticWebController` to build LLM-powered agents that can perceive and interact with web pages. - -## 1. Capturing Page State - -The `captureState()` method provides everything an agent needs to "see" the page. - -```kotlin -scope.launch { - when (val result = controller.captureState()) { - is AgentResult.Success -> { - val state = result.data - // 1. Pass the compact tree to your LLM text prompt - val promptTree = state.compactTree - - // 2. Pass the screenshot to your Vision model (if enabled) - val screenshot = state.screenshotBase64 - - println("Agent is looking at: ${state.title} (${state.url})") - } - is AgentResult.Error -> { - println("Failed to capture state: ${result.error}") - } - } -} -``` - -### The Compact Tree -The `compactTree` is a token-optimized representation of the DOM. Each element has a `[highlightIndex]` that the agent uses to perform actions. - -Example output: -```text -[15] + + + +
Private account value: $42,000
+
+ +
+ + + + diff --git a/test-pages/fixtures/frame.html b/test-pages/fixtures/frame.html new file mode 100644 index 0000000..66131e1 --- /dev/null +++ b/test-pages/fixtures/frame.html @@ -0,0 +1,10 @@ + + +Frame fixture + +

Same-origin frame

+ + + + + diff --git a/test-pages/fixtures/nested-frame.html b/test-pages/fixtures/nested-frame.html new file mode 100644 index 0000000..7297bca --- /dev/null +++ b/test-pages/fixtures/nested-frame.html @@ -0,0 +1,5 @@ + + +Nested frame fixture + + diff --git a/test-pages/server/server.mjs b/test-pages/server/server.mjs new file mode 100644 index 0000000..4c68982 --- /dev/null +++ b/test-pages/server/server.mjs @@ -0,0 +1,29 @@ +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { extname, join, normalize } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('../fixtures/', import.meta.url)); +const port = Number(process.env.AGENTIC_FIXTURE_PORT || 8787); + +createServer(async (request, response) => { + const pathname = new URL(request.url || '/', 'http://localhost').pathname; + const relative = pathname === '/' ? 'complex.html' : pathname.slice(1); + const file = normalize(join(root, relative)); + if (!file.startsWith(root)) { + response.writeHead(403).end('Forbidden'); + return; + } + try { + const metadata = await stat(file); + if (!metadata.isFile()) throw new Error('Not a file'); + const mime = extname(file) === '.html' ? 'text/html; charset=utf-8' : 'text/plain; charset=utf-8'; + response.writeHead(200, { 'content-type': mime, 'cache-control': 'no-store' }); + createReadStream(file).pipe(response); + } catch { + response.writeHead(404).end('Not found'); + } +}).listen(port, '127.0.0.1', () => { + process.stdout.write(`Agentic WebView fixtures: http://127.0.0.1:${port}/\n`); +}); diff --git a/web-injector/LICENSE b/web-injector/LICENSE deleted file mode 100644 index a06f36a..0000000 --- a/web-injector/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2024 Shanto Islam (@shantoislamdev) - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/web-injector/package.json b/web-injector/package.json deleted file mode 100644 index 73c838a..0000000 --- a/web-injector/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "web-injector", - "version": "0.2.1", - "license": "Apache-2.0", - "description": "TypeScript DOM Engine for Agentic WebView", - "main": "dist/agentic_core.min.js", - "scripts": { - "build": "esbuild src/index.ts --bundle --minify --outfile=dist/agentic_core.min.js --format=iife --global-name=__AgenticInternal --target=es2019", - "test": "jest" - }, - "devDependencies": { - "@types/jest": "^29.5.12", - "esbuild": "^0.20.0", - "jest": "^29.7.0", - "jest-environment-jsdom": "^29.7.0", - "ts-jest": "^29.1.2", - "typescript": "^5.3.3" - } -} diff --git a/web-injector/src/bridge.ts b/web-injector/src/bridge.ts deleted file mode 100644 index 1b54d88..0000000 --- a/web-injector/src/bridge.ts +++ /dev/null @@ -1,34 +0,0 @@ -declare global { - interface Window { - AgenticBridge?: { - onDomUpdate(token: string, json: string): void; - onError(token: string, errorJson: string): void; - }; - } -} - -export class Bridge { - private sessionToken: string | null = null; - - public setSessionToken(token: string): void { - this.sessionToken = token; - } - - public notifyDomUpdate(json: string): void { - if (window.AgenticBridge && this.sessionToken) { - (window.AgenticBridge as any).onDomUpdate(this.sessionToken, json); - } - } - - public sendError(message: string, stack?: string): void { - if (window.AgenticBridge && this.sessionToken) { - (window.AgenticBridge as any).onError(this.sessionToken, JSON.stringify({ error: message, stack })); - } - } - - public resolvePromise(promiseId: string, result: string): void { - if (window.AgenticBridge && this.sessionToken) { - (window.AgenticBridge as any).resolvePromise(this.sessionToken, promiseId, result); - } - } -} diff --git a/web-injector/src/buildDomTree.ts b/web-injector/src/buildDomTree.ts deleted file mode 100644 index 0ecb93f..0000000 --- a/web-injector/src/buildDomTree.ts +++ /dev/null @@ -1,667 +0,0 @@ -export interface DomNodeData { - tagName: string; - attributes: Record; - xpath: string; - children: string[]; - isVisible?: boolean; - isTopElement?: boolean; - isInteractive?: boolean; - isInViewport?: boolean; - highlightIndex?: number; - shadowRoot?: boolean; - type?: 'TEXT_NODE'; - text?: string; -} - -export interface BuildDomTreeResult { - rootId: string; - map: Record; - highlightIndexCount: number; - nextId: number; -} - -const MAX_DEPTH = 100; - -const DISTINCT_INTERACTIVE_TAGS = new Set([ - 'a', 'button', 'input', 'select', 'textarea', - 'summary', 'details', 'label', 'option', -]); - -const INTERACTIVE_ROLES = new Set([ - 'button', 'link', 'menuitem', 'menuitemradio', 'menuitemcheckbox', - 'radio', 'checkbox', 'tab', 'switch', 'slider', 'spinbutton', - 'combobox', 'searchbox', 'textbox', 'listbox', 'option', 'scrollbar', -]); - -const INTERACTIVE_CURSORS = new Set([ - 'pointer', 'move', 'text', 'grab', 'grabbing', 'cell', 'copy', 'alias', - 'all-scroll', 'col-resize', 'context-menu', 'crosshair', 'e-resize', - 'ew-resize', 'help', 'n-resize', 'ne-resize', 'nesw-resize', 'ns-resize', - 'nw-resize', 'nwse-resize', 'row-resize', 's-resize', 'se-resize', - 'sw-resize', 'vertical-text', 'w-resize', 'zoom-in', 'zoom-out', -]); - -const NON_INTERACTIVE_CURSORS = new Set([ - 'not-allowed', 'no-drop', 'wait', 'progress', 'initial', 'inherit', -]); - -const INTERACTIVE_ELEMENT_TAGS = new Set([ - 'a', 'button', 'input', 'select', 'textarea', 'details', 'summary', - 'label', 'option', 'optgroup', 'fieldset', 'legend', -]); - -const INTERACTIVE_ROLES_FULL = new Set([ - 'button', 'menu', 'menubar', 'menuitem', 'menuitemradio', - 'menuitemcheckbox', 'radio', 'checkbox', 'tab', 'switch', 'slider', - 'spinbutton', 'combobox', 'searchbox', 'textbox', 'listbox', 'option', - 'scrollbar', -]); - -const ALWAYS_ACCEPT_TAGS = new Set([ - 'body', 'div', 'main', 'article', 'section', 'nav', 'header', 'footer', -]); - -const LEAF_DENY_LIST = new Set([ - 'svg', 'script', 'style', 'link', 'meta', 'noscript', 'template', -]); - -export class BuildDomTreeEngine { - private highlightIndex = 0; - private nextId = 0; - private visitedNodes: WeakSet | null = null; - private domMap: Record = {}; - private xpathCache = new WeakMap(); - - private boundingRects = new WeakMap(); - private clientRects = new WeakMap(); - private computedStyles = new WeakMap(); - - private viewportExpansion: number; - private elementMap: Map; - - constructor(viewportExpansion: number = 0, elementMap: Map) { - this.viewportExpansion = viewportExpansion; - this.elementMap = elementMap; - } - - build(startId?: number, startHighlightIndex?: number): BuildDomTreeResult { - this.highlightIndex = startHighlightIndex ?? 0; - this.nextId = startId ?? 0; - this.visitedNodes = null; - this.domMap = {}; - this.boundingRects = new WeakMap(); - this.clientRects = new WeakMap(); - this.computedStyles = new WeakMap(); - this.xpathCache = new WeakMap(); - - const rootId = this.buildDomTree(document.body); - return { - rootId: rootId || '', - map: this.domMap, - highlightIndexCount: this.highlightIndex, - nextId: this.nextId, - }; - } - - private getCachedBoundingRect(element: Element): DOMRect | null { - if (!element) return null; - if (this.boundingRects.has(element)) return this.boundingRects.get(element)!; - const rect = element.getBoundingClientRect(); - if (rect) this.boundingRects.set(element, rect); - return rect; - } - - private getCachedComputedStyle(element: Element): CSSStyleDeclaration | null { - if (!element) return null; - if (this.computedStyles.has(element)) return this.computedStyles.get(element)!; - const style = window.getComputedStyle(element); - if (style) this.computedStyles.set(element, style); - return style; - } - - private getCachedClientRects(element: Element): DOMRectList | null { - if (!element) return null; - if (this.clientRects.has(element)) return this.clientRects.get(element)!; - const rects = element.getClientRects(); - if (rects) this.clientRects.set(element, rects); - return rects; - } - - private getElementPosition(currentElement: Element): number { - if (!currentElement.parentElement) return 0; - const tagName = currentElement.nodeName.toLowerCase(); - const siblings = Array.from(currentElement.parentElement.children) - .filter(sib => sib.nodeName.toLowerCase() === tagName); - if (siblings.length === 1) return 0; - return siblings.indexOf(currentElement) + 1; - } - - private getXPathTree(element: Element, stopAtBoundary = true): string { - if (this.xpathCache.has(element)) return this.xpathCache.get(element)!; - - const segments: string[] = []; - let current: Element | null = element; - - while (current && current.nodeType === Node.ELEMENT_NODE) { - if ( - stopAtBoundary && - (current.parentNode instanceof ShadowRoot || - current.parentNode instanceof HTMLIFrameElement) - ) { - break; - } - const position = this.getElementPosition(current); - const tagName = current.nodeName.toLowerCase(); - const xpathIndex = position > 0 ? `[${position}]` : ''; - segments.unshift(`${tagName}${xpathIndex}`); - current = current.parentNode as Element; - } - - const result = segments.join('/'); - this.xpathCache.set(element, result); - return result; - } - - private isTextNodeVisible(textNode: Text): boolean { - try { - if (this.viewportExpansion === -1) { - const parent = textNode.parentElement; - if (!parent) return false; - try { - return parent.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }); - } catch { - const style = window.getComputedStyle(parent); - return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; - } - } - - const range = document.createRange(); - range.selectNodeContents(textNode); - const rects = range.getClientRects(); - if (!rects || rects.length === 0) return false; - - let isAnyRectVisible = false; - let isAnyRectInViewport = false; - - for (const rect of rects) { - if (rect.width > 0 && rect.height > 0) { - isAnyRectVisible = true; - if ( - !(rect.bottom < -this.viewportExpansion || - rect.top > window.innerHeight + this.viewportExpansion || - rect.right < -this.viewportExpansion || - rect.left > window.innerWidth + this.viewportExpansion) - ) { - isAnyRectInViewport = true; - break; - } - } - } - - if (!isAnyRectVisible || !isAnyRectInViewport) return false; - - const parent = textNode.parentElement; - if (!parent) return false; - try { - return parent.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }); - } catch { - const style = window.getComputedStyle(parent); - return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; - } - } catch { - return false; - } - } - - private isElementAccepted(element: Element): boolean { - if (!element || !element.tagName) return false; - const tagName = element.tagName.toLowerCase(); - if (ALWAYS_ACCEPT_TAGS.has(tagName)) return true; - return !LEAF_DENY_LIST.has(tagName); - } - - private isElementVisible(element: HTMLElement): boolean { - const style = this.getCachedComputedStyle(element); - return ( - element.offsetWidth > 0 && - element.offsetHeight > 0 && - style?.visibility !== 'hidden' && - style?.display !== 'none' - ); - } - - private isInteractiveElement(element: HTMLElement): boolean { - if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; - - const tagName = element.tagName.toLowerCase(); - const style = this.getCachedComputedStyle(element); - - if (element.tagName.toLowerCase() !== 'html' && style?.cursor && INTERACTIVE_CURSORS.has(style.cursor)) { - return true; - } - - if (INTERACTIVE_ELEMENT_TAGS.has(tagName)) { - if (style?.cursor && NON_INTERACTIVE_CURSORS.has(style.cursor)) return false; - if (element.hasAttribute('disabled') || element.getAttribute('disabled') === 'true' || element.getAttribute('disabled') === '') return false; - if (element.hasAttribute('readonly') || element.getAttribute('readonly') === 'true' || element.getAttribute('readonly') === '') return false; - if ((element as any).disabled) return false; - if ((element as any).readOnly) return false; - if ((element as any).inert) return false; - return true; - } - - const role = element.getAttribute('role'); - const ariaRole = element.getAttribute('aria-role'); - - if (element.getAttribute('contenteditable') === 'true' || element.isContentEditable) return true; - - if ( - element.classList && - (element.classList.contains('button') || - element.classList.contains('dropdown-toggle') || - element.getAttribute('data-index') || - element.getAttribute('data-toggle') === 'dropdown' || - element.getAttribute('aria-haspopup') === 'true') - ) { - return true; - } - - if ( - INTERACTIVE_ELEMENT_TAGS.has(tagName) || - (role && INTERACTIVE_ROLES_FULL.has(role)) || - (ariaRole && INTERACTIVE_ROLES_FULL.has(ariaRole)) - ) { - return true; - } - - try { - const commonMouseAttrs = ['onclick', 'onmousedown', 'onmouseup', 'ondblclick']; - for (const attr of commonMouseAttrs) { - if (element.hasAttribute(attr) || typeof (element as any)[attr] === 'function') return true; - } - } catch { - // ignore - } - - return false; - } - - private isHeuristicallyInteractive(element: HTMLElement): boolean { - if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; - if (!this.isElementVisible(element)) return false; - - const hasInteractiveAttributes = - element.hasAttribute('role') || - element.hasAttribute('tabindex') || - element.hasAttribute('onclick') || - typeof (element as any).onclick === 'function'; - - const hasInteractiveClass = /\b(btn|clickable|menu|item|entry|link)\b/i.test(element.className || ''); - const isInKnownContainer = Boolean(element.closest('button,a,[role="button"],.menu,.dropdown,.list,.toolbar')); - const hasVisibleChildren = [...element.children].some(c => this.isElementVisible(c as HTMLElement)); - const isParentBody = element.parentElement && element.parentElement.isSameNode(document.body); - - return ( - (this.isInteractiveElement(element) || hasInteractiveAttributes || hasInteractiveClass) && - hasVisibleChildren && - isInKnownContainer && - !isParentBody - ); - } - - private isTopElement(element: HTMLElement): boolean { - if (this.viewportExpansion === -1) return true; - - const rects = this.getCachedClientRects(element); - if (!rects || rects.length === 0) return false; - - let isAnyRectInViewport = false; - for (const rect of rects) { - if ( - rect.width > 0 && rect.height > 0 && - !(rect.bottom < -this.viewportExpansion || - rect.top > window.innerHeight + this.viewportExpansion || - rect.right < -this.viewportExpansion || - rect.left > window.innerWidth + this.viewportExpansion) - ) { - isAnyRectInViewport = true; - break; - } - } - if (!isAnyRectInViewport) return false; - - const doc = element.ownerDocument; - if (doc !== window.document) return true; - - const rootNode = element.getRootNode(); - if (rootNode instanceof ShadowRoot) { - const midRect = rects[Math.floor(rects.length / 2)]; - const centerX = midRect.left + midRect.width / 2; - const centerY = midRect.top + midRect.height / 2; - try { - const topEl = rootNode.elementFromPoint(centerX, centerY); - if (!topEl) return false; - let current: Node | null = topEl; - while (current && current !== rootNode) { - if (current === element) return true; - current = (current as Element).parentElement || null; - } - return false; - } catch { - return true; - } - } - - const margin = 5; - const rect = rects[Math.floor(rects.length / 2)]; - const checkPoints = [ - { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }, - { x: rect.left + margin, y: rect.top + margin }, - { x: rect.right - margin, y: rect.bottom - margin }, - ]; - - return checkPoints.some(({ x, y }) => { - try { - const topEl = document.elementFromPoint(x, y); - // If elementFromPoint returns null (e.g., JSDOM), assume element is top - if (!topEl) return true; - let current: Element | null = topEl; - while (current && current !== document.documentElement) { - if (current === element) return true; - current = current.parentElement; - } - return false; - } catch { - return true; - } - }); - } - - private isInExpandedViewport(element: HTMLElement): boolean { - if (this.viewportExpansion === -1) return true; - - const rects = element.getClientRects(); - if (!rects || rects.length === 0) { - const boundingRect = this.getCachedBoundingRect(element); - if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) return false; - return !( - boundingRect.bottom < -this.viewportExpansion || - boundingRect.top > window.innerHeight + this.viewportExpansion || - boundingRect.right < -this.viewportExpansion || - boundingRect.left > window.innerWidth + this.viewportExpansion - ); - } - - for (const rect of rects) { - if (rect.width === 0 || rect.height === 0) continue; - if ( - !(rect.bottom < -this.viewportExpansion || - rect.top > window.innerHeight + this.viewportExpansion || - rect.right < -this.viewportExpansion || - rect.left > window.innerWidth + this.viewportExpansion) - ) { - return true; - } - } - return false; - } - - private isInteractiveCandidate(element: Element): boolean { - if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; - const tagName = element.tagName.toLowerCase(); - if (INTERACTIVE_ELEMENT_TAGS.has(tagName)) return true; - return ( - element.hasAttribute('onclick') || - element.hasAttribute('role') || - element.hasAttribute('tabindex') || - element.hasAttribute('data-action') || - element.getAttribute('contenteditable') === 'true' - ); - } - - private isElementDistinctInteraction(element: HTMLElement): boolean { - if (!element || element.nodeType !== Node.ELEMENT_NODE) return false; - - const tagName = element.tagName.toLowerCase(); - const role = element.getAttribute('role'); - - if (tagName === 'iframe') return true; - if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) return true; - if (role && INTERACTIVE_ROLES.has(role)) return true; - if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') return true; - if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) return true; - if (element.hasAttribute('onclick') || typeof (element as any).onclick === 'function') return true; - - try { - const commonEventAttrs = [ - 'onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', - 'onsubmit', 'onchange', 'oninput', 'onfocus', 'onblur', - ]; - if (commonEventAttrs.some(attr => element.hasAttribute(attr))) return true; - } catch { - // ignore - } - - if (this.isHeuristicallyInteractive(element)) return true; - return false; - } - - private handleHighlighting( - nodeData: DomNodeData, - node: HTMLElement, - isParentHighlighted: boolean, - ): boolean { - if (!nodeData.isInteractive) return false; - - let shouldHighlight = false; - if (!isParentHighlighted) { - shouldHighlight = true; - } else if (this.isElementDistinctInteraction(node)) { - shouldHighlight = true; - } - - if (shouldHighlight) { - nodeData.isInViewport = this.isInExpandedViewport(node); - if (nodeData.isInViewport || this.viewportExpansion === -1) { - nodeData.highlightIndex = this.highlightIndex++; - return true; - } - } - return false; - } - - private buildDomTree( - node: Node, - isParentHighlighted = false, - depth = 0, - ): string | null { - if (!this.visitedNodes) this.visitedNodes = new WeakSet(); - if (depth > MAX_DEPTH) return null; - - if ( - !node || - (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) - ) { - return null; - } - - if (node.nodeType === Node.ELEMENT_NODE && this.visitedNodes.has(node)) return null; - if (node.nodeType === Node.ELEMENT_NODE) this.visitedNodes.add(node); - - // Handle body root - if (node === document.body) { - const nodeData: DomNodeData = { - tagName: 'body', - attributes: {}, - xpath: '/body', - children: [], - }; - for (const child of Array.from(node.childNodes)) { - const childId = this.buildDomTree(child, false, depth + 1); - if (childId) nodeData.children.push(childId); - } - const id = `${this.nextId++}`; - this.domMap[id] = nodeData; - return id; - } - - // Text nodes - if (node.nodeType === Node.TEXT_NODE) { - const textContent = node.textContent?.trim(); - if (!textContent) return null; - const parentEl = (node as Text).parentElement; - if (!parentEl || parentEl.tagName.toLowerCase() === 'script') return null; - - const id = `${this.nextId++}`; - this.domMap[id] = { - tagName: '', - attributes: {}, - xpath: '', - children: [], - type: 'TEXT_NODE', - text: textContent, - isVisible: this.isTextNodeVisible(node as Text), - }; - return id; - } - - // Element nodes - const element = node as HTMLElement; - if (!this.isElementAccepted(element)) return null; - - // Early viewport check - if (this.viewportExpansion !== -1 && !element.shadowRoot) { - const rect = this.getCachedBoundingRect(element); - const style = this.getCachedComputedStyle(element); - const isFixedOrSticky = style && (style.position === 'fixed' || style.position === 'sticky'); - const hasSize = element.offsetWidth > 0 || element.offsetHeight > 0; - - if ( - !rect || - (!isFixedOrSticky && !hasSize && - (rect.bottom < -this.viewportExpansion || - rect.top > window.innerHeight + this.viewportExpansion || - rect.right < -this.viewportExpansion || - rect.left > window.innerWidth + this.viewportExpansion)) - ) { - return null; - } - } - - const nodeData: DomNodeData = { - tagName: element.tagName.toLowerCase(), - attributes: {}, - xpath: this.getXPathTree(element, true), - children: [], - }; - - // Get attributes for interactive candidates - if ( - this.isInteractiveCandidate(element) || - element.tagName.toLowerCase() === 'iframe' || - element.tagName.toLowerCase() === 'body' - ) { - const attributeNames = element.getAttributeNames?.() || []; - for (const name of attributeNames) { - const value = element.getAttribute(name); - if (value !== null) nodeData.attributes[name] = value; - } - } - - // Visibility, interactivity, highlighting - let nodeWasHighlighted = false; - nodeData.isVisible = this.isElementVisible(element); - if (nodeData.isVisible) { - nodeData.isTopElement = this.isTopElement(element); - const role = element.getAttribute('role'); - const isMenuContainer = role === 'menu' || role === 'menubar' || role === 'listbox'; - - if (nodeData.isTopElement || isMenuContainer) { - nodeData.isInteractive = this.isInteractiveElement(element); - nodeWasHighlighted = this.handleHighlighting(nodeData, element, isParentHighlighted); - } - } - - // Even if not interactive, still process children - const tagName = element.tagName.toLowerCase(); - - if (tagName === 'iframe') { - const rect = this.getCachedBoundingRect(element); - if (rect) { - nodeData.attributes['computedHeight'] = String(Math.ceil(rect.height)); - nodeData.attributes['computedWidth'] = String(Math.ceil(rect.width)); - - const shouldSkip = - (rect.width <= 1 && rect.height <= 1) || - rect.left < -1000 || rect.top < -1000; - - const sandbox = element.getAttribute('sandbox'); - const isRestrictiveSandbox = sandbox !== null && !sandbox.includes('allow-same-origin'); - - if (shouldSkip) { - nodeData.attributes['skipped'] = 'invisible-tracking-iframe'; - } else if (isRestrictiveSandbox) { - nodeData.attributes['error'] = 'Cross-origin iframe access blocked by sandbox'; - } else { - try { - const iframeDoc = (element as HTMLIFrameElement).contentDocument || - (element as HTMLIFrameElement).contentWindow?.document; - if (iframeDoc && iframeDoc.childNodes) { - for (const child of Array.from(iframeDoc.childNodes)) { - const childId = this.buildDomTree(child, false, depth + 1); - if (childId) nodeData.children.push(childId); - } - } - } catch (e: any) { - nodeData.attributes['error'] = e.message; - } - } - } - } else if ( - element.isContentEditable || - element.getAttribute('contenteditable') === 'true' || - element.id === 'tinymce' || - element.classList.contains('mce-content-body') || - (tagName === 'body' && element.getAttribute('data-id')?.startsWith('mce_')) - ) { - for (const child of Array.from(element.childNodes)) { - const childId = this.buildDomTree(child, nodeWasHighlighted, depth + 1); - if (childId) nodeData.children.push(childId); - } - } else { - // Shadow DOM - if (element.shadowRoot) { - nodeData.shadowRoot = true; - for (const child of Array.from(element.shadowRoot.childNodes)) { - const childId = this.buildDomTree(child, nodeWasHighlighted, depth + 1); - if (childId) nodeData.children.push(childId); - } - } - // Regular children - for (const child of Array.from(element.childNodes)) { - const passHighlight = nodeWasHighlighted || isParentHighlighted; - const childId = this.buildDomTree(child, passHighlight, depth + 1); - if (childId) nodeData.children.push(childId); - } - } - - // Skip empty anchors - if (nodeData.tagName === 'a' && nodeData.children.length === 0 && !nodeData.attributes.href) { - const rect = this.getCachedBoundingRect(element); - const hasSize = (rect && rect.width > 0 && rect.height > 0) || element.offsetWidth > 0 || element.offsetHeight > 0; - if (!hasSize) return null; - } - - const id = `${this.nextId++}`; - this.domMap[id] = nodeData; - - // Tag element with agent ID for Kotlin-side lookups - if (nodeData.highlightIndex !== undefined && nodeData.highlightIndex !== null) { - const agentId = nodeData.highlightIndex.toString(); - element.setAttribute('data-agent-id', agentId); - this.elementMap.set(agentId, element); - } - - return id; - } -} diff --git a/web-injector/src/cssSelector.ts b/web-injector/src/cssSelector.ts deleted file mode 100644 index a16f148..0000000 --- a/web-injector/src/cssSelector.ts +++ /dev/null @@ -1,93 +0,0 @@ -const SAFE_ATTRIBUTES = new Set([ - 'id', 'name', 'type', 'placeholder', 'aria-label', 'aria-labelledby', - 'aria-describedby', 'role', 'for', 'autocomplete', 'required', 'readonly', - 'alt', 'title', 'src', 'href', 'target', - 'data-id', 'data-qa', 'data-cy', 'data-testid', -]); - -export function convertSimpleXPathToCssSelector(xpath: string): string { - if (!xpath) return ''; - const cleanXpath = xpath.replace(/^\//, ''); - const parts = cleanXpath.split('/'); - const cssParts: string[] = []; - - for (const part of parts) { - if (!part) continue; - - if (part.includes(':') && !part.includes('[')) { - cssParts.push(part.replace(/:/g, '\\:')); - continue; - } - - if (part.includes('[')) { - const bracketIndex = part.indexOf('['); - let basePart = part.substring(0, bracketIndex); - if (basePart.includes(':')) basePart = basePart.replace(/:/g, '\\:'); - const indexPart = part.substring(bracketIndex); - const indices = indexPart.split(']').slice(0, -1).map(i => i.replace('[', '')); - - for (const idx of indices) { - if (/^\d+$/.test(idx)) { - const index = parseInt(idx, 10); - basePart += `:nth-of-type(${index})`; - } else if (idx === 'last()') { - basePart += ':last-of-type'; - } else if (idx.includes('position()')) { - if (idx.includes('>1')) basePart += ':nth-of-type(n+2)'; - } - } - cssParts.push(basePart); - } else { - cssParts.push(part); - } - } - - return cssParts.join(' > '); -} - -export function enhancedCssSelectorForElement( - tagName: string, - xpath: string | null, - attributes: Record, - highlightIndex: number | null, -): string { - try { - if (!xpath) return ''; - - let cssSelector = convertSimpleXPathToCssSelector(xpath); - - // Class names - const classValue = attributes.class; - if (classValue) { - const validClassNamePattern = /^[a-zA-Z_][a-zA-Z0-9_-]*$/; - const classes = classValue.trim().split(/\s+/); - for (const className of classes) { - if (className.trim() && validClassNamePattern.test(className)) { - cssSelector += `.${className}`; - } - } - } - - // Safe attributes - for (const [attribute, value] of Object.entries(attributes)) { - if (attribute === 'class') continue; - if (!attribute.trim()) continue; - if (!SAFE_ATTRIBUTES.has(attribute)) continue; - - const safeAttribute = attribute.replace(':', '\\:'); - if (value === '') { - cssSelector += `[${safeAttribute}]`; - } else if (/["'<>`\n\r\t]/.test(value)) { - const collapsedValue = value.replace(/\s+/g, ' ').trim(); - const safeValue = collapsedValue.replace(/"/g, '\\"'); - cssSelector += `[${safeAttribute}*="${safeValue}"]`; - } else { - cssSelector += `[${safeAttribute}="${value}"]`; - } - } - - return cssSelector; - } catch { - return `${tagName || '*'}[data-agent-id='${highlightIndex}']`; - } -} diff --git a/web-injector/src/domParser.test.ts b/web-injector/src/domParser.test.ts deleted file mode 100644 index 2c09a9b..0000000 --- a/web-injector/src/domParser.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { DomParser } from './domParser'; - -describe('domParser', () => { - let parser: DomParser; - - beforeEach(() => { - document.body.innerHTML = ''; - parser = new DomParser(); - - Object.defineProperty(HTMLElement.prototype, 'offsetWidth', { configurable: true, value: 100 }); - Object.defineProperty(HTMLElement.prototype, 'offsetHeight', { configurable: true, value: 30 }); - - Object.defineProperty(HTMLElement.prototype, 'innerText', { - configurable: true, - get: function() { return this.textContent; } - }); - - // Mock getClientRects for JSDOM - HTMLElement.prototype.getClientRects = function() { - const rect = this.getBoundingClientRect(); - return [rect] as any; - }; - HTMLElement.prototype.getBoundingClientRect = function() { - return { x: 0, y: 0, width: 100, height: 30, top: 0, left: 0, right: 100, bottom: 30, toJSON: () => {} }; - }; - - // Mock Range.getClientRects for text node visibility - const origCreateRange = document.createRange.bind(document); - document.createRange = function() { - const range = origCreateRange(); - range.getClientRects = function() { - return [{ x: 0, y: 0, width: 50, height: 14, top: 0, left: 0, right: 50, bottom: 14 }] as any; - }; - return range; - }; - - // Mock checkVisibility for JSDOM - (Element.prototype as any).checkVisibility = function() { return true; }; - - // Mock elementFromPoint - return the element being tested (matching topmost) - document.elementFromPoint = function(_x: number, _y: number) { - // Return deepest leaf in body to simulate realistic hit testing - const all = document.querySelectorAll('button, a, input, select, textarea'); - if (all.length > 0) { - // Find the element nearest to the center of viewport - return all[all.length - 1]; // last interactive element (deepest in DOM) - } - return document.body; - }; - }); - - it('should generate accessibility tree for simple elements', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - const treeStr = JSON.stringify(tree); - expect(treeStr).toContain('Submit Button'); - expect(treeStr).toContain('BUTTON'); - }); - - it('should detect input elements', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - const treeStr = JSON.stringify(tree); - expect(treeStr).toContain('INPUT'); - }); - - it('should handle nested elements', () => { - document.body.innerHTML = ` -
- Link Text -
- `; - - const { tree } = parser.getAccessibilityTree(); - const treeStr = JSON.stringify(tree); - expect(treeStr).toContain('A'); - }); - - it('should return selectorMap with highlight indices', () => { - document.body.innerHTML = ` - - `; - - const { selectorMap } = parser.getAccessibilityTree(); - expect(selectorMap).toBeDefined(); - expect(Object.keys(selectorMap).length).toBeGreaterThan(0); - }); - - it('should include xpath in nodes', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(tree[0].xpath).toBeDefined(); - expect(typeof tree[0].xpath).toBe('string'); - expect(tree[0].xpath.length).toBeGreaterThan(0); - }); - - it('should include isTopElement and isInteractive flags', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(typeof tree[0].isTopElement).toBe('boolean'); - expect(typeof tree[0].isInteractive).toBe('boolean'); - expect(tree[0].isTopElement).toBe(true); - expect(tree[0].isInteractive).toBe(true); - }); - - it('should assign highlightIndex to interactive elements', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(tree[0].highlightIndex).toBe(0); - }); - - it('should include occluded and inIframe fields', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(typeof tree[0].occluded).toBe('boolean'); - expect(typeof tree[0].inIframe).toBe('boolean'); - }); - - it('should handle empty body', () => { - document.body.innerHTML = ''; - const { tree, truncated } = parser.getAccessibilityTree(); - expect(tree).toEqual([]); - expect(truncated).toBe(false); - }); - - it('should include cssSelector in nodes', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(typeof tree[0].cssSelector).toBe('string'); - expect(tree[0].cssSelector.length).toBeGreaterThan(0); - }); - - it('should mark elements as new on first capture', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(tree[0].isNew).toBe(true); - }); - - it('should mark elements as not new on second capture without changes', () => { - document.body.innerHTML = ` - - `; - - parser.getAccessibilityTree(); - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(tree[0].isNew).toBe(false); - }); - - it('should include depth field in nodes', () => { - document.body.innerHTML = ` -
- -
- `; - - const { tree } = parser.getAccessibilityTree(); - expect(tree.length).toBeGreaterThan(0); - expect(typeof tree[0].depth).toBe('number'); - expect(tree[0].depth).toBeGreaterThanOrEqual(0); - }); - - it('should return maxNodeId and maxHighlightIndex', () => { - document.body.innerHTML = ` - - `; - - const result = parser.getAccessibilityTree(); - expect(typeof result.maxNodeId).toBe('number'); - expect(typeof result.maxHighlightIndex).toBe('number'); - expect(result.maxNodeId).toBeGreaterThan(0); - expect(result.maxHighlightIndex).toBeGreaterThan(0); - }); - - it('should collect text from deeply nested non-interactive elements', () => { - document.body.innerHTML = ` - - `; - - const { tree } = parser.getAccessibilityTree(); - const btn = tree.find(n => n.tag === 'BUTTON'); - expect(btn).toBeDefined(); - expect(btn!.text).toContain('Deep Text'); - }); -}); diff --git a/web-injector/src/domParser.ts b/web-injector/src/domParser.ts deleted file mode 100644 index 9a78aa8..0000000 --- a/web-injector/src/domParser.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { BuildDomTreeEngine, DomNodeData, BuildDomTreeResult } from './buildDomTree'; -import { enhancedCssSelectorForElement } from './cssSelector'; - -export interface ElementBounds { - left: number; - top: number; - width: number; - height: number; -} - -export interface AccessibilityNode { - id: string; - tag: string; - text: string; - role: string; - bounds: ElementBounds; - attributes: { [key: string]: string }; - occluded: boolean; - inIframe: boolean; - xpath: string; - isTopElement: boolean; - isInteractive: boolean; - highlightIndex: number | null; - cssSelector: string; - isNew: boolean; - depth: number; -} - -export interface AccessibilityTreeResult { - tree: AccessibilityNode[]; - truncated: boolean; - selectorMap: Record; - maxNodeId: number; - maxHighlightIndex: number; -} - -function quickHash(node: AccessibilityNode): string { - const branchPath = (node.xpath || '').replace(/\[\d+\]/g, ''); - const attrs = Object.entries(node.attributes) - .filter(([k]) => k !== 'data-agent-id') - .sort(([a], [b]) => a.localeCompare(b)) - .map(([k, v]) => `${k}=${v}`) - .join('|'); - let h1 = 0, h2 = 0; - for (let i = 0; i < branchPath.length; i++) { - h1 = ((h1 << 5) - h1 + branchPath.charCodeAt(i)) | 0; - } - for (let i = 0; i < attrs.length; i++) { - h2 = ((h2 << 5) - h2 + attrs.charCodeAt(i)) | 0; - } - return `${h1}-${h2}-${node.xpath || ''}`; -} - -export class DomParser { - private elementMap = new Map(); - private engine: BuildDomTreeEngine; - private viewportExpansion: number; - private subframeTrees: Map = new Map(); - private previousElementHashes: Map = new Map(); - - constructor(viewportExpansion: number = 0) { - this.viewportExpansion = viewportExpansion; - this.engine = new BuildDomTreeEngine(this.viewportExpansion, this.elementMap); - } - - public getAccessibilityTree(maxElements: number = 500): AccessibilityTreeResult { - this.subframeTrees.clear(); - this.elementMap.clear(); - this.engine = new BuildDomTreeEngine(this.viewportExpansion, this.elementMap); - - const result = this.engine.build(); - const nodes: AccessibilityNode[] = []; - const selectorMap: Record = {}; - - this.processNodeMap(result.map, result.rootId, nodes, selectorMap, false, maxElements, 0); - - // Merge subframe trees - for (const [url, subframeNodes] of this.subframeTrees) { - for (const node of subframeNodes) { - if (nodes.length >= maxElements) break; - node.inIframe = true; - nodes.push(node); - if (node.highlightIndex !== null && node.highlightIndex !== undefined) { - selectorMap[node.highlightIndex] = node.id; - } - } - } - - // Compute isNew flag via hash diffing - const currentHashes = new Map(); - for (const node of nodes) { - if (node.highlightIndex !== null && node.highlightIndex !== undefined) { - const hash = quickHash(node); - currentHashes.set(node.highlightIndex, hash); - node.isNew = !this.previousElementHashes.has(node.highlightIndex) - || this.previousElementHashes.get(node.highlightIndex) !== hash; - } - } - this.previousElementHashes = currentHashes; - - this.pruneElementMap(); - - // Compute max IDs for cross-frame offset management - let maxNodeId = 0; - for (const id of Object.keys(result.map)) { - const numId = parseInt(id, 10); - if (!isNaN(numId) && numId > maxNodeId) maxNodeId = numId; - } - const maxHighlightIndex = result.highlightIndexCount; - - return { - tree: nodes, - truncated: nodes.length >= maxElements, - selectorMap, - maxNodeId, - maxHighlightIndex, - }; - } - - private processNodeMap( - map: Record, - nodeId: string, - nodes: AccessibilityNode[], - selectorMap: Record, - inIframe: boolean, - maxElements: number, - depth: number, - ): void { - if (nodes.length >= maxElements) return; - - const nodeData = map[nodeId]; - if (!nodeData) return; - - // Process element nodes - if (nodeData.tagName && nodeData.type !== 'TEXT_NODE') { - // Include interactive elements (ones with highlightIndex) - if (nodeData.highlightIndex !== undefined && nodeData.highlightIndex !== null) { - const agentId = nodeData.highlightIndex.toString(); - const el = this.elementMap.get(agentId); - - const cssSelector = enhancedCssSelectorForElement( - nodeData.tagName, nodeData.xpath, nodeData.attributes, nodeData.highlightIndex - ); - - const node: AccessibilityNode = { - id: agentId, - tag: nodeData.tagName.toUpperCase(), - text: this.extractText(nodeData, map), - role: nodeData.attributes['role'] || '', - bounds: el ? this.getBounds(el) : { left: 0, top: 0, width: 0, height: 0 }, - attributes: nodeData.attributes, - occluded: el ? !nodeData.isTopElement! : false, - inIframe, - xpath: nodeData.xpath, - isTopElement: nodeData.isTopElement || false, - isInteractive: nodeData.isInteractive || false, - highlightIndex: nodeData.highlightIndex, - cssSelector, - isNew: false, - depth, - }; - - nodes.push(node); - selectorMap[nodeData.highlightIndex] = agentId; - } - // Also include visible, top-element nodes without highlightIndex - // (for backward compatibility and non-interactive element visibility) - else if ( - nodeData.isVisible && - nodeData.isTopElement && - !nodeData.isInteractive && - nodeData.children.length > 0 - ) { - // Don't add to nodes list, but recurse into children - } - - // Recurse into children - for (const childId of nodeData.children) { - this.processNodeMap(map, childId, nodes, selectorMap, inIframe, maxElements, depth + 1); - } - } - } - - private extractText(nodeData: DomNodeData, map: Record): string { - const textParts: string[] = []; - const collectText = (data: DomNodeData) => { - for (const childId of data.children) { - const child = map[childId]; - if (!child) continue; - if (child.type === 'TEXT_NODE' && child.text && child.isVisible) { - textParts.push(child.text); - } else if (child.tagName && child.type !== 'TEXT_NODE') { - if (child.highlightIndex === undefined || child.highlightIndex === null) { - collectText(child); - } - } - } - }; - collectText(nodeData); - if (textParts.length > 0) return textParts.join(' ').trim(); - - // Fallback to attributes - return nodeData.attributes['aria-label'] || - nodeData.attributes['placeholder'] || - nodeData.attributes['title'] || - nodeData.attributes['value'] || - ''; - } - - private getBounds(el: Element): ElementBounds { - const rect = el.getBoundingClientRect(); - return { - left: rect.left, - top: rect.top, - width: rect.width, - height: rect.height, - }; - } - - public getElementById(id: string): Element | undefined { - return this.elementMap.get(id); - } - - public getSelectorMap(): Map { - const result = new Map(); - for (const [agentId, _] of this.elementMap) { - const numId = parseInt(agentId, 10); - if (!isNaN(numId)) result.set(numId, agentId); - } - return result; - } - - public mergeSubframeTree(url: string, nodes: AccessibilityNode[]): void { - this.subframeTrees.set(url, nodes); - } - - private pruneElementMap(): void { - for (const [id, el] of this.elementMap.entries()) { - if (!document.body.contains(el) && !el.isConnected) { - this.elementMap.delete(id); - } - } - } -} diff --git a/web-injector/src/elementHash.ts b/web-injector/src/elementHash.ts deleted file mode 100644 index b5f0406..0000000 --- a/web-injector/src/elementHash.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { AccessibilityNode } from './domParser'; - -async function sha256(input: string): Promise { - const encoder = new TextEncoder(); - const data = encoder.encode(input); - const hashBuffer = await crypto.subtle.digest('SHA-256', data); - const hashArray = Array.from(new Uint8Array(hashBuffer)); - return hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); -} - -function getParentBranchPath(node: AccessibilityNode, allNodes: AccessibilityNode[]): string[] { - const path: string[] = []; - // Walk up via id chain - simplified: use xpath segments - if (node.xpath) { - const segments = node.xpath.split('/'); - for (const seg of segments) { - const tag = seg.replace(/\[\d+\]/, ''); - if (tag) path.push(tag); - } - } - return path; -} - -export async function hashDomElement(node: AccessibilityNode): Promise<{ - branchPathHash: string; - attributesHash: string; - xpathHash: string; -}> { - const branchPath = getParentBranchPath(node, []); - const branchPathStr = branchPath.join('/'); - const attributesStr = Object.entries(node.attributes) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([k, v]) => `${k}=${v}`) - .join(''); - - const [branchPathHash, attributesHash, xpathHash] = await Promise.all([ - sha256(branchPathStr), - sha256(attributesStr), - sha256(node.xpath || ''), - ]); - - return { branchPathHash, attributesHash, xpathHash }; -} - -export async function hashDomElementQuick(node: AccessibilityNode): Promise { - const { branchPathHash, attributesHash, xpathHash } = await hashDomElement(node); - return `${branchPathHash}-${attributesHash}-${xpathHash}`; -} - -export async function findElementInTree( - targetHash: { branchPathHash: string; attributesHash: string; xpathHash: string }, - nodes: AccessibilityNode[], -): Promise { - for (const node of nodes) { - if (node.highlightIndex === null || node.highlightIndex === undefined) continue; - const hash = await hashDomElement(node); - if ( - hash.branchPathHash === targetHash.branchPathHash && - hash.attributesHash === targetHash.attributesHash && - hash.xpathHash === targetHash.xpathHash - ) { - return node; - } - } - return null; -} diff --git a/web-injector/src/iframeBus.ts b/web-injector/src/iframeBus.ts deleted file mode 100644 index 0aff0b8..0000000 --- a/web-injector/src/iframeBus.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { DomParser } from './domParser'; - -interface IframeTreeMessage { - type: '__agentic_iframe_tree'; - url: string; - tree: string; // JSON-stringified AccessibilityNode[] - maxNodeId?: number; - maxHighlightIndex?: number; -} - -export class IframeMessageBus { - private parser: DomParser; - private isTopFrame: boolean; - - constructor(parser: DomParser) { - this.parser = parser; - this.isTopFrame = window.top === window; - - if (this.isTopFrame) { - this.listenForSubframeTrees(); - } else { - this.relayToParent(); - } - } - - private listenForSubframeTrees(): void { - window.addEventListener('message', (event: MessageEvent) => { - if (event.data?.type === '__agentic_iframe_tree') { - const msg = event.data as IframeTreeMessage; - try { - this.parser.mergeSubframeTree(msg.url, JSON.parse(msg.tree)); - } catch { - // ignore parse errors - } - } - }); - } - - private relayToParent(): void { - // Subframe: listen for request from parent and send tree - window.addEventListener('message', (event: MessageEvent) => { - if (event.data?.type === '__agentic_request_tree') { - this.sendTreeToParent(); - } - }); - - // Auto-send tree on load - if (document.readyState === 'complete') { - setTimeout(() => this.sendTreeToParent(), 100); - } else { - window.addEventListener('load', () => { - setTimeout(() => this.sendTreeToParent(), 100); - }); - } - } - - private sendTreeToParent(): void { - try { - const result = this.parser.getAccessibilityTree(); - const msg: IframeTreeMessage = { - type: '__agentic_iframe_tree', - url: location.href, - tree: JSON.stringify(result.tree), - maxNodeId: result.maxNodeId, - maxHighlightIndex: result.maxHighlightIndex, - }; - window.parent.postMessage(msg, '*'); - } catch { - // ignore errors - } - } - - requestSubframeTrees(): void { - if (!this.isTopFrame) return; - const iframes = document.querySelectorAll('iframe'); - for (const iframe of iframes) { - try { - iframe.contentWindow?.postMessage({ type: '__agentic_request_tree' }, '*'); - } catch { - // cross-origin, can't post - } - } - } -} diff --git a/web-injector/src/index.ts b/web-injector/src/index.ts deleted file mode 100644 index 6978511..0000000 --- a/web-injector/src/index.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { DomParser } from './domParser'; -import { InteractionHandler } from './interaction'; -import { Bridge } from './bridge'; -import { IframeMessageBus } from './iframeBus'; -import { serializeTreeToText } from './serializer'; -import { waitForElementStability } from './stability'; - -const parser = new DomParser(); -const interaction = new InteractionHandler(); -const bridge = new Bridge(); - -// ─── Runtime Config ────────────────────────────────────────────── -let runtimeConfig = { - viewportExpansion: 0, - domMutationThrottleMs: 300, - enableAntiDetection: true, - includeAttributes: null as string[] | null, -}; - -// ─── Engine Object ───────────────────────────────────────────────── -// CRITICAL: Assign to window FIRST, before any initialization code -// that could throw (MutationObserver, IframeMessageBus, anti-detection). -// If initialization crashes, at least the engine API is available. - -export const AgenticEngine = { - setSessionToken(token: string) { - try { bridge.setSessionToken(token); } catch (e) { /* ignore */ } - }, - - getAccessibilityTree(maxElements?: number) { - try { - return JSON.stringify(parser.getAccessibilityTree(maxElements)); - } catch (e) { - return JSON.stringify({ tree: [], truncated: false, selectorMap: {} }); - } - }, - - getViewportInfo() { - try { - const vvScale = window.visualViewport ? window.visualViewport.scale : 1; - return JSON.stringify({ - devicePixelRatio: window.devicePixelRatio || 1, - visualViewportScale: vvScale, - scrollX: window.scrollX || 0, - scrollY: window.scrollY || 0, - viewportWidth: window.innerWidth || 0, - viewportHeight: window.innerHeight || 0, - }); - } catch (e) { - return JSON.stringify({ - devicePixelRatio: 1, visualViewportScale: 1, - scrollX: 0, scrollY: 0, viewportWidth: 0, viewportHeight: 0, - }); - } - }, - - getSelectorMap() { - try { - const { selectorMap } = parser.getAccessibilityTree(); - return JSON.stringify(selectorMap); - } catch (e) { - return '{}'; - } - }, - - getCompactTree(maxElements?: number) { - try { - const { tree } = parser.getAccessibilityTree(maxElements); - return serializeTreeToText(tree); - } catch (e) { - return ''; - } - }, - - configure(configJson: string) { - try { - const cfg = JSON.parse(configJson); - if (cfg.viewportExpansion !== undefined) runtimeConfig.viewportExpansion = cfg.viewportExpansion; - if (cfg.domMutationThrottleMs !== undefined) runtimeConfig.domMutationThrottleMs = cfg.domMutationThrottleMs; - if (cfg.enableAntiDetection !== undefined) runtimeConfig.enableAntiDetection = cfg.enableAntiDetection; - if (cfg.includeAttributes !== undefined) runtimeConfig.includeAttributes = cfg.includeAttributes; - } catch (e) { /* ignore */ } - }, - - getFullCapture(maxElements?: number) { - try { - const result = parser.getAccessibilityTree(maxElements); - const viewportInfo = JSON.parse(AgenticEngine.getViewportInfo()); - const compactTree = serializeTreeToText( - result.tree, - runtimeConfig.includeAttributes, - undefined, - { - scrollY: viewportInfo.scrollY || 0, - scrollHeight: document.documentElement?.scrollHeight || 0, - viewportHeight: viewportInfo.viewportHeight || window.innerHeight || 0, - }, - false - ); - return JSON.stringify({ - tree: result.tree, - truncated: result.truncated, - selectorMap: result.selectorMap, - compactTree, - maxNodeId: result.maxNodeId, - maxHighlightIndex: result.maxHighlightIndex, - }); - } catch (e) { - return JSON.stringify({ - tree: [], truncated: false, selectorMap: {}, compactTree: '', - maxNodeId: 0, maxHighlightIndex: 0, - }); - } - }, - - async scrollIntoView(agentId: string, promiseId: string) { - try { - const el = parser.getElementById(agentId); - if (el) { - await interaction.scrollIntoViewAndWait(el); - bridge.resolvePromise(promiseId, 'true'); - return true; - } - bridge.resolvePromise(promiseId, 'false'); - return false; - } catch (e) { - try { bridge.resolvePromise(promiseId, 'false'); } catch (_e) { /* ignore */ } - return false; - } - }, - - setInputValue(agentId: string, text: string) { - try { - const el = parser.getElementById(agentId); - if (el instanceof HTMLElement) { - return interaction.setInputValue(el, text); - } - return false; - } catch (e) { - return false; - } - }, - - setSelectOption(agentId: string, value: string) { - try { - const el = parser.getElementById(agentId); - if (el instanceof HTMLSelectElement) { - return interaction.setSelectValue(el, value); - } - return false; - } catch (e) { - return false; - } - }, - - getElementCenter(agentId: string) { - try { - const el = parser.getElementById(agentId); - if (el) { - return JSON.stringify(interaction.getPhysicalCenter(el)); - } - return null; - } catch (e) { - return null; - } - }, - - sendKeys(keys: string) { - try { return interaction.sendKeys(keys); } catch (e) { return false; } - }, - - scrollToPercent(yPercent: number, agentId?: string) { - try { - const el = agentId ? parser.getElementById(agentId) : undefined; - interaction.scrollToPercent(yPercent, el); - return true; - } catch (e) { return false; } - }, - - scrollToTop(agentId?: string) { - try { - const el = agentId ? parser.getElementById(agentId) : undefined; - interaction.scrollToTop(el); - return true; - } catch (e) { return false; } - }, - - scrollToBottom(agentId?: string) { - try { - const el = agentId ? parser.getElementById(agentId) : undefined; - interaction.scrollToBottom(el); - return true; - } catch (e) { return false; } - }, - - previousPage(agentId?: string) { - try { - const el = agentId ? parser.getElementById(agentId) : undefined; - interaction.previousPage(el); - return true; - } catch (e) { return false; } - }, - - nextPage(agentId?: string) { - try { - const el = agentId ? parser.getElementById(agentId) : undefined; - interaction.nextPage(el); - return true; - } catch (e) { return false; } - }, - - scrollToText(text: string, nth: number = 0) { - try { return interaction.scrollToText(text, nth); } catch (e) { return false; } - }, - - getDropdownOptions(agentId: string) { - try { - const el = parser.getElementById(agentId); - if (el) { - return JSON.stringify(interaction.getDropdownOptions(el)); - } - return '[]'; - } catch (e) { - return '[]'; - } - }, - - selectDropdownOption(agentId: string, text: string) { - try { - const el = parser.getElementById(agentId); - if (el) { - return interaction.selectDropdownOption(el, text); - } - return false; - } catch (e) { - return false; - } - }, - - async waitForStability(agentId: string, timeoutMs: number = 1000) { - try { - const el = parser.getElementById(agentId); - if (!el) return false; - return waitForElementStability(el, timeoutMs); - } catch (e) { - return false; - } - }, - - dismissDialogs() { - // Placeholder — Kotlin side handles via WebChromeClient overrides. - }, - - isFileUploader(agentId: string) { - try { - const el = parser.getElementById(agentId); - if (el) return interaction.isFileUploader(el); - return false; - } catch (e) { return false; } - }, -}; - -// ─── Assign to window IMMEDIATELY ────────────────────────────────── -// This MUST happen before any code that could throw (MutationObserver, -// IframeMessageBus, anti-detection). If any of that crashes, the -// engine API is still available for Kotlin to call. -(window as any).__AgenticInternal = AgenticEngine; - -// ─── Post-assignment initialization (safe to crash) ──────────────── - -function injectAntiDetection(): void { - if (!runtimeConfig.enableAntiDetection) return; - try { - Object.defineProperty(navigator, 'webdriver', { get: function() { return undefined; } }); - } catch (e) { /* ignore */ } - try { - (window as any).chrome = { runtime: {} }; - } catch (e) { /* ignore */ } - try { - const origAttachShadow = Element.prototype.attachShadow; - Element.prototype.attachShadow = function (opts: ShadowRootInit) { - return origAttachShadow.call(this, Object.assign({}, opts, { mode: 'open' })); - }; - } catch (e) { /* ignore */ } -} - -function setupMutationObserverSafe(): void { - if (!document.body) { - // Body not available yet — defer until DOMContentLoaded - document.addEventListener('DOMContentLoaded', function() { - setupMutationObserverSafe(); - }, { once: true }); - return; - } - try { - interaction.setupMutationObserver(function() { - try { - var result = parser.getAccessibilityTree(); - bridge.notifyDomUpdate(JSON.stringify({ tree: result.tree, selectorMap: result.selectorMap })); - } catch (e) { /* ignore mutation observer errors */ } - }, runtimeConfig.domMutationThrottleMs); - } catch (e) { - // MutationObserver setup failed — non-fatal, bridge updates won't fire - } -} - -// Initialize IframeMessageBus (safe — only adds event listeners) -try { - new IframeMessageBus(parser); -} catch (e) { /* ignore */ } - -injectAntiDetection(); -setupMutationObserverSafe(); diff --git a/web-injector/src/interaction.ts b/web-injector/src/interaction.ts deleted file mode 100644 index f974277..0000000 --- a/web-injector/src/interaction.ts +++ /dev/null @@ -1,259 +0,0 @@ -export class InteractionHandler { - public getPhysicalCenter(el: Element): { x: number; y: number } { - const rect = el.getBoundingClientRect(); - const dpr = window.devicePixelRatio; - const zoom = window.visualViewport?.scale ?? 1; - return { - x: (rect.left + rect.width / 2) * dpr * zoom, - y: (rect.top + rect.height / 2) * dpr * zoom, - }; - } - - public async scrollIntoViewAndWait(el: Element): Promise { - return new Promise(resolve => { - el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'smooth' }); - - const observer = new IntersectionObserver(entries => { - if (entries[0].isIntersecting) { - observer.disconnect(); - requestAnimationFrame(() => requestAnimationFrame(resolve)); - } - }, { threshold: 0.5 }); - - observer.observe(el); - - setTimeout(() => { - observer.disconnect(); - resolve(); - }, 2000); - }); - } - - public setInputValue(el: HTMLElement, text: string): boolean { - const dispatchFrameworkEvents = (target: HTMLElement, val: string) => { - const beforeInputEvent = new InputEvent('beforeinput', { - bubbles: true, cancelable: true, inputType: 'insertText', data: val - }); - target.dispatchEvent(beforeInputEvent); - - const inputEvent = new InputEvent('input', { - bubbles: true, cancelable: true, inputType: 'insertText', data: val - }); - target.dispatchEvent(inputEvent); - - target.dispatchEvent(new Event('change', { bubbles: true })); - }; - - if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { - const nativeSetter = Object.getOwnPropertyDescriptor( - el instanceof HTMLInputElement ? HTMLInputElement.prototype : HTMLTextAreaElement.prototype, - 'value' - )?.set; - - if (nativeSetter) { - nativeSetter.call(el, text); - dispatchFrameworkEvents(el, text); - return true; - } - } - - if (el.isContentEditable) { - el.innerText = text; - dispatchFrameworkEvents(el, text); - return true; - } - - return false; - } - - public setSelectValue(el: HTMLSelectElement, value: string): boolean { - el.value = value; - el.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - - public sendKeys(keys: string): boolean { - const keyParts = keys.split('+').map(k => k.trim()); - const modifiers = keyParts.slice(0, -1); - const mainKey = keyParts[keyParts.length - 1]; - - const modifierMap: Record = { - 'ctrl': 'Control', 'control': 'Control', - 'shift': 'Shift', - 'alt': 'Alt', - 'meta': 'Meta', 'cmd': 'Meta', 'command': 'Meta', - }; - - const activeModifiers: string[] = []; - for (const mod of modifiers) { - const mapped = modifierMap[mod.toLowerCase()] || mod; - activeModifiers.push(mapped); - document.activeElement?.dispatchEvent(new KeyboardEvent('keydown', { - key: mapped, code: mapped === 'Control' ? 'ControlLeft' : mapped, - bubbles: true, cancelable: true, - })); - } - - document.activeElement?.dispatchEvent(new KeyboardEvent('keydown', { - key: mainKey, code: mainKey, bubbles: true, cancelable: true, - })); - document.activeElement?.dispatchEvent(new KeyboardEvent('keyup', { - key: mainKey, code: mainKey, bubbles: true, cancelable: true, - })); - - for (const mod of activeModifiers.reverse()) { - document.activeElement?.dispatchEvent(new KeyboardEvent('keyup', { - key: mod, code: mod === 'Control' ? 'ControlLeft' : mod, - bubbles: true, cancelable: true, - })); - } - - return true; - } - - public scrollToPercent(yPercent: number, el?: Element): void { - if (el) { - const scrollable = this.findNearestScrollableElement(el); - if (scrollable) { - const maxScroll = scrollable.scrollHeight - scrollable.clientHeight; - scrollable.scrollTo({ top: maxScroll * (yPercent / 100), behavior: 'smooth' }); - return; - } - } - const maxScroll = document.documentElement.scrollHeight - window.innerHeight; - window.scrollTo({ top: maxScroll * (yPercent / 100), behavior: 'smooth' }); - } - - public scrollToTop(el?: Element): void { - if (el) { - const scrollable = this.findNearestScrollableElement(el); - if (scrollable) { scrollable.scrollTo({ top: 0, behavior: 'smooth' }); return; } - } - window.scrollTo({ top: 0, behavior: 'smooth' }); - } - - public scrollToBottom(el?: Element): void { - if (el) { - const scrollable = this.findNearestScrollableElement(el); - if (scrollable) { scrollable.scrollTo({ top: scrollable.scrollHeight, behavior: 'smooth' }); return; } - } - window.scrollTo({ top: document.documentElement.scrollHeight, behavior: 'smooth' }); - } - - public previousPage(el?: Element): void { - if (el) { - const scrollable = this.findNearestScrollableElement(el); - if (scrollable) { scrollable.scrollBy({ top: -scrollable.clientHeight, behavior: 'smooth' }); return; } - } - window.scrollBy({ top: -window.innerHeight, behavior: 'smooth' }); - } - - public nextPage(el?: Element): void { - if (el) { - const scrollable = this.findNearestScrollableElement(el); - if (scrollable) { scrollable.scrollBy({ top: scrollable.clientHeight, behavior: 'smooth' }); return; } - } - window.scrollBy({ top: window.innerHeight, behavior: 'smooth' }); - } - - public scrollToText(text: string, nth: number = 0): boolean { - const walker = document.createTreeWalker( - document.body, - NodeFilter.SHOW_TEXT, - { - acceptNode: (node) => - node.textContent?.toLowerCase().includes(text.toLowerCase()) - ? NodeFilter.FILTER_ACCEPT - : NodeFilter.FILTER_REJECT, - } - ); - - let count = 0; - let node: Text | null; - while ((node = walker.nextNode() as Text | null)) { - if (count === nth) { - const parent = node.parentElement; - if (parent) { - parent.scrollIntoView({ behavior: 'auto', block: 'center' }); - return true; - } - } - count++; - } - return false; - } - - public getDropdownOptions(el: Element): Array<{ value: string; text: string; index: number }> { - if (!(el instanceof HTMLSelectElement)) return []; - return Array.from(el.options).map((opt, i) => ({ - value: opt.value, - text: opt.textContent?.trim() || opt.label || opt.value, - index: i, - })); - } - - public selectDropdownOption(el: Element, text: string): boolean { - if (!(el instanceof HTMLSelectElement)) return false; - - const lowerText = text.toLowerCase(); - for (const opt of Array.from(el.options)) { - if ( - opt.value.toLowerCase() === lowerText || - (opt.textContent?.trim().toLowerCase() || '') === lowerText || - (opt.label?.toLowerCase() || '') === lowerText - ) { - el.value = opt.value; - el.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - } - return false; - } - - public findNearestScrollableElement(el: Element): Element | null { - let current: Element | null = el; - while (current && current !== document.documentElement) { - const style = window.getComputedStyle(current); - const overflowY = style.overflowY; - if ( - (overflowY === 'scroll' || overflowY === 'auto') && - current.scrollHeight > current.clientHeight - ) { - return current; - } - current = current.parentElement; - } - return document.scrollingElement || document.documentElement; - } - - public isFileUploader(el: Element): boolean { - if (el instanceof HTMLInputElement) { - return el.type === 'file' || el.hasAttribute('accept'); - } - return false; - } - - public setupMutationObserver(callback: () => void, throttleMs: number = 300): MutationObserver { - let timeout: any = null; - const observer = new MutationObserver(() => { - if (timeout) return; - timeout = setTimeout(() => { - callback(); - timeout = null; - }, throttleMs); - }); - - observer.observe(document.body, { - childList: true, - subtree: true, - attributes: true, - attributeFilter: [ - 'class', 'style', 'hidden', 'disabled', - 'aria-hidden', 'aria-disabled', 'readonly', - 'checked', 'selected', 'src', 'href' - ] - }); - - return observer; - } -} diff --git a/web-injector/src/serializer.ts b/web-injector/src/serializer.ts deleted file mode 100644 index 038e870..0000000 --- a/web-injector/src/serializer.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { AccessibilityNode } from './domParser'; - -const DEFAULT_INCLUDE_ATTRIBUTES = [ - 'title', 'type', 'checked', 'name', 'role', 'value', 'placeholder', - 'data-date-format', 'data-state', 'alt', 'aria-checked', 'aria-label', - 'aria-expanded', 'href', -]; - -function capTextLength(text: string, maxLength: number): string { - if (text.length <= maxLength) return text; - return text.substring(0, maxLength) + '...'; -} - -export function serializeTreeToText( - nodes: AccessibilityNode[], - includeAttributes: string[] | null = null, - previousHighlightIndices?: Set, - scrollInfo?: { scrollY: number; scrollHeight: number; viewportHeight: number }, - hierarchical: boolean = false, -): string { - if (!includeAttributes) includeAttributes = DEFAULT_INCLUDE_ATTRIBUTES; - const lines: string[] = []; - - if (scrollInfo) { - const maxScroll = Math.max(1, scrollInfo.scrollHeight - scrollInfo.viewportHeight); - const pct = Math.round((scrollInfo.scrollY / maxScroll) * 100); - lines.push(`[Scroll info] scrollY: ${scrollInfo.scrollY}, scrollHeight: ${scrollInfo.scrollHeight}, viewportHeight: ${scrollInfo.viewportHeight}, scrollPercent: ${pct}%`); - } - - for (const node of nodes) { - if (node.highlightIndex === null || node.highlightIndex === undefined) continue; - - const depthStr = hierarchical ? '\t'.repeat(node.depth ?? 0) : ''; - const text = node.text || ''; - - let attributesHtmlStr: string | null = null; - const attributesToInclude: Record = {}; - - for (const [key, value] of Object.entries(node.attributes)) { - if (includeAttributes.includes(key) && String(value).trim() !== '') { - attributesToInclude[key] = String(value).trim(); - } - } - - // Dedup attribute values - const orderedKeys = includeAttributes.filter(key => key in attributesToInclude); - if (orderedKeys.length > 1) { - const keysToRemove = new Set(); - const seenValues: Record = {}; - for (const key of orderedKeys) { - const value = attributesToInclude[key]; - if (value.length > 5) { - if (value in seenValues) { - keysToRemove.add(key); - } else { - seenValues[value] = key; - } - } - } - for (const key of keysToRemove) delete attributesToInclude[key]; - } - - // Remove role if it matches tag - if (node.tag && node.tag.toLowerCase() === attributesToInclude.role) { - delete attributesToInclude.role; - } - - // Remove attributes that duplicate text - const attrsToRemoveIfTextMatches = ['aria-label', 'placeholder', 'title']; - for (const attr of attrsToRemoveIfTextMatches) { - if ( - attributesToInclude[attr] && - attributesToInclude[attr].trim().toLowerCase() === text.trim().toLowerCase() - ) { - delete attributesToInclude[attr]; - } - } - - if (Object.keys(attributesToInclude).length > 0) { - attributesHtmlStr = Object.entries(attributesToInclude) - .map(([key, value]) => `${key}=${capTextLength(value, 15)}`) - .join(' '); - } - - const isNew = previousHighlightIndices && !previousHighlightIndices.has(node.highlightIndex); - const highlightIndicator = isNew ? `*[${node.highlightIndex}]` : `[${node.highlightIndex}]`; - - let line = `${depthStr}${highlightIndicator}<${node.tag.toLowerCase()}`; - - if (attributesHtmlStr) { - line += ` ${attributesHtmlStr}`; - } - - if (text) { - const trimmedText = text.trim(); - if (!attributesHtmlStr) line += ' '; - line += `>${trimmedText}`; - } else if (!attributesHtmlStr) { - line += ' '; - } - - line += ' />'; - lines.push(line); - } - - return lines.join('\n'); -} diff --git a/web-injector/src/stability.ts b/web-injector/src/stability.ts deleted file mode 100644 index 0ba7307..0000000 --- a/web-injector/src/stability.ts +++ /dev/null @@ -1,44 +0,0 @@ -export async function waitForElementStability( - el: Element, - timeoutMs: number = 1000, - pollIntervalMs: number = 50, -): Promise { - const startTime = performance.now(); - let lastRect: DOMRect | null = null; - let stableCount = 0; - const STABLE_THRESHOLD = 2; // px - const REQUIRED_STABLE_CYCLES = 1; - - return new Promise((resolve) => { - const check = () => { - const rect = el.getBoundingClientRect(); - if (lastRect) { - const dx = Math.abs(rect.x - lastRect.x); - const dy = Math.abs(rect.y - lastRect.y); - const dw = Math.abs(rect.width - lastRect.width); - const dh = Math.abs(rect.height - lastRect.height); - - if (dx < STABLE_THRESHOLD && dy < STABLE_THRESHOLD && - dw < STABLE_THRESHOLD && dh < STABLE_THRESHOLD) { - stableCount++; - if (stableCount >= REQUIRED_STABLE_CYCLES) { - resolve(true); - return; - } - } else { - stableCount = 0; - } - } - lastRect = rect; - - if (performance.now() - startTime >= timeoutMs) { - resolve(false); - return; - } - - setTimeout(check, pollIntervalMs); - }; - - check(); - }); -} diff --git a/web-injector/.gitignore b/web-runtime/.gitignore similarity index 100% rename from web-injector/.gitignore rename to web-runtime/.gitignore diff --git a/agentic-webview/LICENSE b/web-runtime/LICENSE similarity index 100% rename from agentic-webview/LICENSE rename to web-runtime/LICENSE diff --git a/web-injector/jest.config.js b/web-runtime/jest.config.js similarity index 100% rename from web-injector/jest.config.js rename to web-runtime/jest.config.js diff --git a/web-injector/package-lock.json b/web-runtime/package-lock.json similarity index 99% rename from web-injector/package-lock.json rename to web-runtime/package-lock.json index eab678e..6ec35f1 100644 --- a/web-injector/package-lock.json +++ b/web-runtime/package-lock.json @@ -1,12 +1,12 @@ { - "name": "web-injector", - "version": "0.2.1", + "name": "agentic-webview-runtime", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "web-injector", - "version": "0.2.1", + "name": "agentic-webview-runtime", + "version": "0.3.0", "license": "Apache-2.0", "devDependencies": { "@types/jest": "^29.5.12", diff --git a/web-runtime/package.json b/web-runtime/package.json new file mode 100644 index 0000000..f088df1 --- /dev/null +++ b/web-runtime/package.json @@ -0,0 +1,21 @@ +{ + "name": "agentic-webview-runtime", + "version": "0.3.0", + "license": "Apache-2.0", + "description": "Versioned page-side runtime for Agentic WebView", + "main": "dist/agentic_runtime.min.js", + "scripts": { + "build": "esbuild src/index.ts --bundle --minify --outfile=dist/agentic_runtime.min.js --format=iife --global-name=AgenticWebRuntimeBundle --target=es2019", + "typecheck": "tsc --noEmit", + "test": "jest", + "check": "npm run typecheck && npm test -- --runInBand && npm run build" + }, + "devDependencies": { + "@types/jest": "^29.5.12", + "esbuild": "^0.20.0", + "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", + "ts-jest": "^29.1.2", + "typescript": "^5.3.3" + } +} diff --git a/web-runtime/src/index.ts b/web-runtime/src/index.ts new file mode 100644 index 0000000..b2f4a3b --- /dev/null +++ b/web-runtime/src/index.ts @@ -0,0 +1,85 @@ +import { RuntimeProtocolDispatcher } from './protocol/dispatcher'; +import { RUNTIME_PROTOCOL_VERSION, RuntimeMethods } from './protocol/types'; +import { SemanticRuntime } from './runtime/semanticRuntime'; +import { SemanticObservationOptions } from './runtime/semanticObserver'; + +const bootstrapConfiguration = window.__AgenticWebRuntimeBootstrapConfiguration; +const semanticRuntime = new SemanticRuntime(); +if (bootstrapConfiguration) { + semanticRuntime.configure(bootstrapConfiguration); + delete window.__AgenticWebRuntimeBootstrapConfiguration; +} + +const protocolDispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: (_payload, request) => { + semanticRuntime.activate(request.documentId); + return { + ready: true, + protocolVersion: RUNTIME_PROTOCOL_VERSION, + capabilities: semanticRuntime.capabilities(), + }; + }, + [RuntimeMethods.configure]: (payload, request) => { + semanticRuntime.activate(request.documentId); + semanticRuntime.configure(payload); + return { configured: true }; + }, + [RuntimeMethods.captureObservation]: (payload, request) => + semanticRuntime.capture(request.documentId, payload as Partial), + [RuntimeMethods.executeAction]: (payload, request) => { + const command = payload.command; + if (typeof command !== 'object' || command === null || Array.isArray(command)) { + throw new Error('action.execute requires a command object'); + } + const options = typeof payload.options === 'object' && payload.options !== null && !Array.isArray(payload.options) + ? payload.options as Record + : {}; + return semanticRuntime.execute(request.documentId, command as Record, options); + }, + [RuntimeMethods.prepareNativeClick]: (payload, request) => + semanticRuntime.prepareNativeClick( + request.documentId, + payload.target, + typeof payload.options === 'object' && payload.options !== null && !Array.isArray(payload.options) + ? payload.options as Record + : {}, + ), + [RuntimeMethods.verifyNativeClick]: (payload, request) => { + if (typeof payload.token !== 'string' || !payload.token.trim()) { + throw new Error('action.verify_native_click requires a token'); + } + return semanticRuntime.verifyNativeClick(request.documentId, payload.token); + }, +}, runtimeMessageLimit(bootstrapConfiguration)); + +export const AgenticWebRuntime = Object.freeze({ + protocolVersion: RUNTIME_PROTOCOL_VERSION, + dispatchProtocol(requestJson: string): Promise { + return protocolDispatcher.dispatch(requestJson); + }, +}); + +declare global { + interface Window { + __AgenticWebRuntime?: typeof AgenticWebRuntime; + __AgenticWebRuntimeBootstrapConfiguration?: Record; + } +} + +Object.defineProperty(window, '__AgenticWebRuntime', { + value: AgenticWebRuntime, + writable: false, + configurable: false, + enumerable: false, +}); + +function runtimeMessageLimit(configuration: Record | undefined): number { + const runtime = configuration?.runtime; + if (typeof runtime !== 'object' || runtime === null || Array.isArray(runtime)) { + return 2 * 1024 * 1024; + } + const value = (runtime as Record).maximumMessageBytes; + return typeof value === 'number' && Number.isInteger(value) && value >= 1024 && value <= 16 * 1024 * 1024 + ? value + : 2 * 1024 * 1024; +} diff --git a/web-runtime/src/protocol/dispatcher.test.ts b/web-runtime/src/protocol/dispatcher.test.ts new file mode 100644 index 0000000..ba86e61 --- /dev/null +++ b/web-runtime/src/protocol/dispatcher.test.ts @@ -0,0 +1,119 @@ +import { RuntimeHandlerError, RuntimeProtocolDispatcher } from './dispatcher'; +import { RUNTIME_PROTOCOL_VERSION, RuntimeMethods, RuntimeResponseEnvelope } from './types'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +function request(overrides: Record = {}): string { + return JSON.stringify({ + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: 'bridge-token-12345678901234567890123456789012', + sessionId: 'session-1', + documentId: 'document-1', + requestId: 'request-1', + method: RuntimeMethods.ping, + payload: {}, + ...overrides, + }); +} + +describe('RuntimeProtocolDispatcher', () => { + it('matches the shared version-one ping fixtures', async () => { + const fixture = (name: string): string => readFileSync( + resolve(process.cwd(), '..', 'protocol-fixtures', 'v1', name), + 'utf8', + ); + const dispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: () => ({ ready: true }), + }); + + const actual = JSON.parse(await dispatcher.dispatch(fixture('request-ping.json'))); + const expected = JSON.parse(fixture('response-ping-success.json')); + + expect(actual).toEqual(expected); + }); + it('dispatches a valid request and preserves correlation fields', async () => { + const dispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: () => ({ ready: true }), + }); + + const response = JSON.parse(await dispatcher.dispatch(request())) as RuntimeResponseEnvelope; + + expect(response).toEqual({ + protocolVersion: 1, + bridgeToken: 'bridge-token-12345678901234567890123456789012', + sessionId: 'session-1', + documentId: 'document-1', + requestId: 'request-1', + status: 'success', + result: { ready: true }, + }); + }); + + it('rejects unknown methods with a structured error', async () => { + const dispatcher = new RuntimeProtocolDispatcher(); + + const response = JSON.parse(await dispatcher.dispatch(request())) as RuntimeResponseEnvelope; + + expect(response.status).toBe('error'); + expect(response.error?.code).toBe('METHOD_NOT_FOUND'); + }); + + it('rejects protocol version mismatches', async () => { + const dispatcher = new RuntimeProtocolDispatcher(); + + const response = JSON.parse(await dispatcher.dispatch(request({ protocolVersion: 99 }))) as RuntimeResponseEnvelope; + + expect(response.status).toBe('error'); + expect(response.error?.code).toBe('PROTOCOL_VERSION_MISMATCH'); + expect(response.requestId).toBe('request-1'); + }); + + it('rejects malformed JSON without throwing', async () => { + const dispatcher = new RuntimeProtocolDispatcher(); + + const response = JSON.parse(await dispatcher.dispatch('{broken')) as RuntimeResponseEnvelope; + + expect(response.status).toBe('error'); + expect(response.error?.code).toBe('MALFORMED_JSON'); + expect(response.requestId).toBe('unknown'); + }); + + it('maps expected handler failures without losing details', async () => { + const dispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: () => { + throw new RuntimeHandlerError('NOT_READY', 'Runtime is not ready', { phase: 'initializing' }); + }, + }); + + const response = JSON.parse(await dispatcher.dispatch(request())) as RuntimeResponseEnvelope; + + expect(response.status).toBe('error'); + expect(response.error).toEqual({ + code: 'NOT_READY', + message: 'Runtime is not ready', + details: { phase: 'initializing' }, + }); + }); + + it('returns a correlated error when a handler result exceeds the response budget', async () => { + const dispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: () => ({ value: 'x'.repeat(2_000) }), + }, 1_024); + + const response = JSON.parse(await dispatcher.dispatch(request())) as RuntimeResponseEnvelope; + + expect(response.requestId).toBe('request-1'); + expect(response.status).toBe('error'); + expect(response.error?.code).toBe('RESPONSE_TOO_LARGE'); + }); + + it('rejects duplicate handler registration', () => { + const dispatcher = new RuntimeProtocolDispatcher({ + [RuntimeMethods.ping]: () => null, + }); + + expect(() => dispatcher.register(RuntimeMethods.ping, () => null)).toThrow( + 'Runtime handler already registered', + ); + }); +}); diff --git a/web-runtime/src/protocol/dispatcher.ts b/web-runtime/src/protocol/dispatcher.ts new file mode 100644 index 0000000..e993ce1 --- /dev/null +++ b/web-runtime/src/protocol/dispatcher.ts @@ -0,0 +1,173 @@ +import { + RUNTIME_PROTOCOL_VERSION, + RuntimeMethodHandler, + RuntimeRequestEnvelope, + RuntimeResponseEnvelope, +} from './types'; +import { + encodeResponseEnvelope, + parseRequestEnvelope, + ProtocolValidationError, +} from './validation'; + +export class RuntimeHandlerError extends Error { + public readonly code: string; + public readonly details: Record; + + constructor(code: string, message: string, details: Record = {}) { + super(message); + this.name = 'RuntimeHandlerError'; + this.code = code; + this.details = details; + } +} + +export class RuntimeProtocolDispatcher { + private readonly handlers = new Map(); + + constructor( + handlers: Record = {}, + private readonly maximumMessageBytes: number = 2 * 1024 * 1024, + ) { + if (!Number.isInteger(maximumMessageBytes) || maximumMessageBytes < 1024) { + throw new Error('maximumMessageBytes must be an integer of at least 1024'); + } + for (const [method, handler] of Object.entries(handlers)) { + this.register(method, handler); + } + } + + public register(method: string, handler: RuntimeMethodHandler): void { + if (this.handlers.has(method)) { + throw new Error(`Runtime handler already registered for ${method}`); + } + this.handlers.set(method, handler); + } + + public async dispatch(raw: string): Promise { + let request: RuntimeRequestEnvelope; + try { + request = parseRequestEnvelope(raw, this.maximumMessageBytes); + } catch (error) { + return this.encodeValidationFailure(raw, error); + } + + const handler = this.handlers.get(request.method); + if (!handler) { + return this.encodeSafely(errorResponse(request, { + code: 'METHOD_NOT_FOUND', + message: `No runtime handler is registered for ${request.method}`, + details: { method: request.method }, + })); + } + + try { + const result = await handler(request.payload, request); + return this.encodeSafely({ + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: request.bridgeToken, + sessionId: request.sessionId, + documentId: request.documentId, + requestId: request.requestId, + status: 'success', + result: result === undefined ? null : result, + }); + } catch (error) { + if (error instanceof RuntimeHandlerError) { + return this.encodeSafely(errorResponse(request, { + code: error.code, + message: boundedMessage(error.message), + details: error.details, + })); + } + return this.encodeSafely(errorResponse(request, { + code: 'INTERNAL_ERROR', + message: boundedMessage(error instanceof Error ? error.message : 'Unknown runtime error'), + details: {}, + })); + } + } + + private encodeSafely(response: RuntimeResponseEnvelope): string { + try { + return encodeResponseEnvelope(response, this.maximumMessageBytes); + } catch { + return encodeResponseEnvelope({ + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: response.bridgeToken, + sessionId: response.sessionId, + documentId: response.documentId, + requestId: response.requestId, + status: 'error', + error: { + code: 'RESPONSE_TOO_LARGE', + message: 'Runtime response exceeded the configured message limit', + details: {}, + }, + }, this.maximumMessageBytes); + } + } + + private encodeValidationFailure(raw: string, error: unknown): string { + const identifiers = extractUntrustedIdentifiers(raw); + const protocolError = error instanceof ProtocolValidationError + ? { code: error.code, message: error.message, details: error.details } + : { code: 'INVALID_REQUEST', message: 'Runtime request validation failed', details: {} }; + + return this.encodeSafely({ + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: identifiers.bridgeToken, + sessionId: identifiers.sessionId, + documentId: identifiers.documentId, + requestId: identifiers.requestId, + status: 'error', + error: protocolError, + }); + } +} + +function boundedMessage(message: string): string { + const normalized = message.trim() || 'Runtime operation failed'; + return normalized.length <= 2_000 ? normalized : `${normalized.slice(0, 1_999)}…`; +} + +function errorResponse( + request: RuntimeRequestEnvelope, + error: { code: string; message: string; details: Record }, +): RuntimeResponseEnvelope { + return { + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: request.bridgeToken, + sessionId: request.sessionId, + documentId: request.documentId, + requestId: request.requestId, + status: 'error', + error, + }; +} + +function extractUntrustedIdentifiers(raw: string): { + sessionId: string; + bridgeToken: string; + documentId: string; + requestId: string; +} { + const fallback = { bridgeToken: 'unknown', sessionId: 'unknown', documentId: 'unknown', requestId: 'unknown' }; + try { + const value = JSON.parse(raw) as Record; + return { + bridgeToken: safeIdentifier(value?.bridgeToken) ?? fallback.bridgeToken, + sessionId: safeIdentifier(value?.sessionId) ?? fallback.sessionId, + documentId: safeIdentifier(value?.documentId) ?? fallback.documentId, + requestId: safeIdentifier(value?.requestId) ?? fallback.requestId, + }; + } catch { + return fallback; + } +} + +function safeIdentifier(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 && value.length <= 256 + ? value + : null; +} diff --git a/web-runtime/src/protocol/types.ts b/web-runtime/src/protocol/types.ts new file mode 100644 index 0000000..6d09dcf --- /dev/null +++ b/web-runtime/src/protocol/types.ts @@ -0,0 +1,44 @@ +export const RUNTIME_PROTOCOL_VERSION = 1 as const; + +export type RuntimeResponseStatus = 'success' | 'error'; + +export interface RuntimeRequestEnvelope { + protocolVersion: number; + bridgeToken: string; + sessionId: string; + documentId: string; + requestId: string; + method: string; + payload: Record; +} + +export interface RuntimeProtocolError { + code: string; + message: string; + details: Record; +} + +export interface RuntimeResponseEnvelope { + protocolVersion: number; + bridgeToken: string; + sessionId: string; + documentId: string; + requestId: string; + status: RuntimeResponseStatus; + result?: unknown; + error?: RuntimeProtocolError; +} + +export type RuntimeMethodHandler = ( + payload: Record, + request: RuntimeRequestEnvelope, +) => unknown | Promise; + +export const RuntimeMethods = { + ping: 'system.ping', + configure: 'runtime.configure', + captureObservation: 'observation.capture', + executeAction: 'action.execute', + prepareNativeClick: 'action.prepare_native_click', + verifyNativeClick: 'action.verify_native_click', +} as const; diff --git a/web-runtime/src/protocol/validation.ts b/web-runtime/src/protocol/validation.ts new file mode 100644 index 0000000..289832e --- /dev/null +++ b/web-runtime/src/protocol/validation.ts @@ -0,0 +1,140 @@ +import { + RUNTIME_PROTOCOL_VERSION, + RuntimeRequestEnvelope, + RuntimeResponseEnvelope, +} from './types'; + +const METHOD_PATTERN = /^[a-z][a-z0-9]*(?:\.[a-z][a-z0-9_]*)+$/; + +export class ProtocolValidationError extends Error { + public readonly code: string; + public readonly details: Record; + + constructor(code: string, message: string, details: Record = {}) { + super(message); + this.name = 'ProtocolValidationError'; + this.code = code; + this.details = details; + } +} + +export function parseRequestEnvelope(raw: string, maximumMessageBytes: number): RuntimeRequestEnvelope { + const byteCount = utf8ByteLength(raw); + if (byteCount > maximumMessageBytes) { + throw new ProtocolValidationError( + 'MESSAGE_TOO_LARGE', + `Runtime request is ${byteCount} bytes; limit is ${maximumMessageBytes}`, + { byteCount, maximumMessageBytes }, + ); + } + + let candidate: unknown; + try { + candidate = JSON.parse(raw); + } catch { + throw new ProtocolValidationError('MALFORMED_JSON', 'Runtime request is not valid JSON'); + } + + if (!isRecord(candidate)) { + throw new ProtocolValidationError('INVALID_ENVELOPE', 'Runtime request must be a JSON object'); + } + + const protocolVersion = requireInteger(candidate.protocolVersion, 'protocolVersion'); + if (protocolVersion !== RUNTIME_PROTOCOL_VERSION) { + throw new ProtocolValidationError( + 'PROTOCOL_VERSION_MISMATCH', + `Expected protocol version ${RUNTIME_PROTOCOL_VERSION} but received ${protocolVersion}`, + { expected: RUNTIME_PROTOCOL_VERSION, actual: protocolVersion }, + ); + } + + const method = requireNonBlankString(candidate.method, 'method'); + if (!METHOD_PATTERN.test(method)) { + throw new ProtocolValidationError('INVALID_METHOD', 'Runtime method has an invalid format'); + } + + const payload = candidate.payload === undefined ? {} : candidate.payload; + if (!isRecord(payload)) { + throw new ProtocolValidationError('INVALID_PAYLOAD', 'Runtime payload must be a JSON object'); + } + + return { + protocolVersion, + bridgeToken: requireSecret(candidate.bridgeToken, 'bridgeToken'), + sessionId: requireNonBlankString(candidate.sessionId, 'sessionId'), + documentId: requireNonBlankString(candidate.documentId, 'documentId'), + requestId: requireNonBlankString(candidate.requestId, 'requestId'), + method, + payload, + }; +} + +export function encodeResponseEnvelope( + response: RuntimeResponseEnvelope, + maximumMessageBytes: number, +): string { + const encoded = JSON.stringify(response); + const byteCount = utf8ByteLength(encoded); + if (byteCount > maximumMessageBytes) { + const fallback: RuntimeResponseEnvelope = { + protocolVersion: RUNTIME_PROTOCOL_VERSION, + bridgeToken: response.bridgeToken, + sessionId: response.sessionId, + documentId: response.documentId, + requestId: response.requestId, + status: 'error', + error: { + code: 'RESPONSE_TOO_LARGE', + message: `Runtime response exceeded the ${maximumMessageBytes} byte limit`, + details: { byteCount, maximumMessageBytes }, + }, + }; + const fallbackEncoded = JSON.stringify(fallback); + if (utf8ByteLength(fallbackEncoded) > maximumMessageBytes) { + throw new ProtocolValidationError( + 'MESSAGE_LIMIT_TOO_SMALL', + 'Message limit is too small to encode a protocol error', + ); + } + return fallbackEncoded; + } + return encoded; +} + +function requireNonBlankString(value: unknown, field: string): string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ProtocolValidationError('INVALID_ENVELOPE', `${field} must be a non-blank string`); + } + return value; +} + +function requireSecret(value: unknown, field: string): string { + const secret = requireNonBlankString(value, field); + if (secret.length < 32 || secret.length > 256) { + throw new ProtocolValidationError('INVALID_ENVELOPE', `${field} must contain 32..256 characters`); + } + return secret; +} + +function requireInteger(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isInteger(value)) { + throw new ProtocolValidationError('INVALID_ENVELOPE', `${field} must be an integer`); + } + return value; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function utf8ByteLength(value: string): number { + let bytes = 0; + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint <= 0x7f) bytes += 1; + else if (codePoint <= 0x7ff) bytes += 2; + else if (codePoint <= 0xffff) bytes += 3; + else bytes += 4; + } + return bytes; +} diff --git a/web-runtime/src/runtime/actionExecutor.test.ts b/web-runtime/src/runtime/actionExecutor.test.ts new file mode 100644 index 0000000..241ae98 --- /dev/null +++ b/web-runtime/src/runtime/actionExecutor.test.ts @@ -0,0 +1,232 @@ +import { ElementRegistry } from './elementRegistry'; +import { DocumentRevisionTracker } from './revisionTracker'; +import { SemanticActionExecutor } from './actionExecutor'; + +describe('SemanticActionExecutor', () => { + let registry: ElementRegistry; + let revisions: DocumentRevisionTracker; + let executor: SemanticActionExecutor; + + beforeEach(() => { + document.body.innerHTML = ''; + registry = new ElementRegistry(); + revisions = new DocumentRevisionTracker(); + executor = new SemanticActionExecutor(registry, revisions, () => 'document-1'); + }); + + it('types through the native setter and verifies the resulting value', async () => { + const input = document.createElement('input'); + document.body.appendChild(input); + const target = reference(registry.getOrCreate(input)); + + const receipt = await executor.execute('document-1', { + type: 'type_text', + target, + text: 'Hello', + mode: 'REPLACE_ALL', + }); + + expect(input.value).toBe('Hello'); + expect(receipt.dispatched).toBe(true); + expect(receipt.verified).toBe(true); + expect(receipt.strategy).toBe('DOM_NATIVE_SETTER'); + }); + + it.each([ + ['APPEND', 'Start', ' plus', 'Start plus'], + ['CLEAR', 'Start', 'ignored', ''], + ])('supports %s text mode', async (mode, initial, text, expected) => { + const input = document.createElement('input'); + input.value = initial; + document.body.appendChild(input); + + const receipt = await executor.execute('document-1', { + type: 'type_text', + target: reference(registry.getOrCreate(input)), + text, + mode, + }); + + expect(input.value).toBe(expected); + expect(receipt.verified).toBe(true); + }); + + it('inserts text at the current selection', async () => { + const input = document.createElement('input'); + input.value = 'abcd'; + document.body.appendChild(input); + input.setSelectionRange(1, 3); + + await executor.execute('document-1', { + type: 'type_text', + target: reference(registry.getOrCreate(input)), + text: 'X', + mode: 'INSERT_AT_SELECTION', + }); + + expect(input.value).toBe('aXd'); + }); + + it('rejects detached element references as stale', async () => { + const button = document.createElement('button'); + document.body.appendChild(button); + const target = reference(registry.getOrCreate(button)); + button.remove(); + + await expect(executor.execute('document-1', { type: 'click', target })) + .rejects.toMatchObject({ code: 'STALE_ELEMENT' }); + }); + + it('verifies a click only when it produces an observable effect', async () => { + const button = document.createElement('button'); + button.addEventListener('click', () => button.setAttribute('aria-pressed', 'true')); + document.body.appendChild(button); + + const receipt = await executor.execute('document-1', { + type: 'click', + target: reference(registry.getOrCreate(button)), + }); + + expect(receipt.dispatched).toBe(true); + expect(receipt.verified).toBe(true); + }); + + it('reports an effectless click as dispatched but unverified', async () => { + const button = document.createElement('button'); + document.body.appendChild(button); + + const receipt = await executor.execute('document-1', { + type: 'click', + target: reference(registry.getOrCreate(button)), + }); + + expect(receipt.dispatched).toBe(true); + expect(receipt.verified).toBe(false); + }); + + it('selects options by label and verifies the value', async () => { + document.body.innerHTML = ''; + const select = document.querySelector('select')!; + const target = reference(registry.getOrCreate(select)); + + const receipt = await executor.execute('document-1', { + type: 'select_option', + target, + option: { type: 'label', label: 'Beta' }, + }); + + expect(select.value).toBe('b'); + expect(receipt.verified).toBe(true); + }); + + it('rejects disabled controls before dispatch', async () => { + const button = document.createElement('button'); + button.disabled = true; + document.body.appendChild(button); + const target = reference(registry.getOrCreate(button)); + + await expect(executor.execute('document-1', { type: 'click', target })) + .rejects.toMatchObject({ code: 'ELEMENT_NOT_ACTIONABLE' }); + }); + + it('rejects references whose observation revision is in the future', async () => { + const button = document.createElement('button'); + document.body.appendChild(button); + const target = { ...reference(registry.getOrCreate(button)), observedAtRevision: 1 }; + + await expect(executor.execute('document-1', { type: 'click', target })) + .rejects.toMatchObject({ code: 'STALE_ELEMENT' }); + }); + + it('types into a validated same-origin frame element', async () => { + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + const input = iframe.contentDocument!.createElement('input'); + iframe.contentDocument!.body.appendChild(input); + const elementId = registry.getOrCreate(input, 'main:f1'); + const framedExecutor = new SemanticActionExecutor( + registry, + revisions, + () => 'document-1', + (_root, frameId, documentId) => + frameId === 'main:f1' && documentId === 'document-1#main:f1', + ); + + const receipt = await framedExecutor.execute('document-1', { + type: 'type_text', + target: { + documentId: 'document-1#main:f1', + frameId: 'main:f1', + elementId, + observedAtRevision: 0, + }, + text: 'Frame text', + mode: 'REPLACE_ALL', + }); + + expect(input.value).toBe('Frame text'); + expect(receipt.verified).toBe(true); + }); + + it('dispatches and verifies a long press without retrying', async () => { + const button = document.createElement('button'); + button.addEventListener('pointerup', () => button.setAttribute('aria-expanded', 'true')); + document.body.appendChild(button); + + const receipt = await executor.execute('document-1', { + type: 'long_press', + target: reference(registry.getOrCreate(button)), + durationMs: 1, + }); + + expect(receipt.dispatched).toBe(true); + expect(receipt.verified).toBe(true); + expect(receipt.strategy).toBe('DOM_POINTER'); + }); + + it('dispatches a key chord to the active element', async () => { + const input = document.createElement('input'); + input.addEventListener('keydown', () => { input.value = 'handled'; }); + document.body.appendChild(input); + input.focus(); + + const receipt = await executor.execute('document-1', { + type: 'press_keys', + chord: { key: 'a', control: true, alt: false, shift: false, meta: false }, + }); + + expect(receipt.verified).toBe(true); + expect(receipt.strategy).toBe('DOM_KEYBOARD'); + }); + + it('prepares and verifies an externally dispatched native click', async () => { + const button = document.createElement('button'); + button.getBoundingClientRect = () => ({ + x: 10, y: 20, left: 10, top: 20, width: 100, height: 40, + right: 110, bottom: 60, toJSON: () => ({}), + } as DOMRect); + button.addEventListener('click', () => button.setAttribute('aria-pressed', 'true')); + document.body.appendChild(button); + const target = reference(registry.getOrCreate(button)); + + const preparation = await executor.prepareNativeClick( + 'document-1', + target, + () => ({ left: 0, top: 0 }), + ); + button.click(); + const receipt = executor.verifyNativeClick('document-1', preparation.token as string); + + expect(receipt.strategy).toBe('ANDROID_NATIVE_POINTER'); + expect(receipt.verified).toBe(true); + }); + + function reference(elementId: string) { + return { + documentId: 'document-1', + frameId: 'main', + elementId, + observedAtRevision: 0, + }; + } +}); diff --git a/web-runtime/src/runtime/actionExecutor.ts b/web-runtime/src/runtime/actionExecutor.ts new file mode 100644 index 0000000..0949561 --- /dev/null +++ b/web-runtime/src/runtime/actionExecutor.ts @@ -0,0 +1,636 @@ +import { RuntimeHandlerError } from '../protocol/dispatcher'; +import { ElementRegistry } from './elementRegistry'; +import { DocumentRevisionTracker } from './revisionTracker'; + +interface ElementReferencePayload { + documentId: string; + frameId: string; + elementId: string; + observedAtRevision: number; +} + +interface CommandReceipt { + commandType: string; + strategy: string; + documentId: string; + revisionBefore: number; + revisionAfter: number; + dispatched: boolean; + verified: boolean; + pageChanged: boolean; +} + +interface ActionExecutionOptions { + geometryStableCycles: number; + geometryTolerancePx: number; +} + +const DEFAULT_ACTION_OPTIONS: ActionExecutionOptions = { + geometryStableCycles: 2, + geometryTolerancePx: 1, +}; + +export class SemanticActionExecutor { + private nextNativePointerToken = 1; + private readonly nativePointerPreparations = new Map(); + constructor( + private readonly registry: ElementRegistry, + private readonly revisions: DocumentRevisionTracker, + private readonly activeDocumentId: () => string | null, + private readonly validatesReference: ( + rootDocumentId: string, + frameId: string, + documentId: string, + ) => boolean = (rootDocumentId, frameId, documentId) => + frameId === 'main' && documentId === rootDocumentId, + ) {} + + public async execute( + documentId: string, + command: Record, + requestedOptions: Partial = {}, + ): Promise { + if (documentId !== this.activeDocumentId()) { + throw new RuntimeHandlerError('STALE_DOCUMENT', 'Command belongs to an inactive document'); + } + + const type = requireString(command.type, 'command.type'); + const options = actionOptions(requestedOptions); + const revisionBefore = this.revisions.current; + const urlBefore = location.href; + let dispatched = false; + let verified = false; + let strategy = 'DOM_POINTER'; + + switch (type) { + case 'click': { + const element = this.resolveTarget(documentId, command.target); + requireEnabled(element); + const before = verificationSnapshot(element, this.revisions); + scrollElementIntoView(element, 'center'); + await waitForStableGeometry(element, options); + requireNotOccluded(element); + const button = pointerButton(command.button); + let clickObserved = false; + const verificationEvent = button === 2 ? 'contextmenu' : 'click'; + element.addEventListener(verificationEvent, () => { clickObserved = true; }, { once: true, capture: true }); + dispatchPointerSequence(element, button, false); + dispatched = true; + await settleMutationDelivery(); + verified = clickObserved && hasObservableEffect(before, element, this.revisions); + break; + } + case 'long_press': { + const element = this.resolveTarget(documentId, command.target); + requireEnabled(element); + const before = verificationSnapshot(element, this.revisions); + scrollElementIntoView(element, 'center'); + await waitForStableGeometry(element, options); + requireNotOccluded(element); + const durationMs = requireFiniteNumber(command.durationMs, 'command.durationMs'); + if (!Number.isInteger(durationMs) || durationMs < 1 || durationMs > 60_000) { + throw new RuntimeHandlerError('INVALID_ACTION', 'command.durationMs must be an integer within 1..60000'); + } + let downObserved = false; + let upObserved = false; + element.addEventListener('pointerdown', () => { downObserved = true; }, { once: true, capture: true }); + element.addEventListener('pointerup', () => { upObserved = true; }, { once: true, capture: true }); + dispatchPointerDown(element, 0); + dispatched = true; + await delay(durationMs); + dispatchPointerUp(element, 0); + await settleMutationDelivery(); + verified = downObserved && upObserved && hasObservableEffect(before, element, this.revisions); + break; + } + case 'type_text': { + strategy = 'DOM_NATIVE_SETTER'; + const element = this.resolveTarget(documentId, command.target); + requireEnabled(element); + const text = requireString(command.text, 'command.text', true); + const mode = requireString(command.mode, 'command.mode'); + const expected = setText(element, text, mode); + dispatched = true; + await settleMutationDelivery(); + verified = currentTextValue(element) === expected; + break; + } + case 'select_option': { + strategy = 'DOM_SELECT'; + const element = this.resolveTarget(documentId, command.target); + if (element.tagName.toLowerCase() !== 'select') { + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Target is not a select element'); + } + const select = element as HTMLSelectElement; + requireEnabled(element); + const option = requireRecord(command.option, 'command.option'); + const selected = selectOption(select, option); + const view = element.ownerDocument.defaultView ?? window; + select.value = selected.value; + select.dispatchEvent(new view.Event('input', { bubbles: true })); + select.dispatchEvent(new view.Event('change', { bubbles: true })); + dispatched = true; + await settleMutationDelivery(); + verified = select.value === selected.value; + break; + } + case 'scroll': { + strategy = 'DOM_SCROLL'; + const delta = requireRecord(command.delta, 'command.delta'); + const x = requireFiniteNumber(delta.xCssPx, 'command.delta.xCssPx'); + const y = requireFiniteNumber(delta.yCssPx, 'command.delta.yCssPx'); + const target = requireRecord(command.target, 'command.target'); + const targetType = requireString(target.type, 'command.target.type'); + const scrollContainer = targetType === 'element' + ? nearestScrollable(this.resolveTarget(documentId, target.target)) + : document.scrollingElement || document.documentElement; + const beforeX = scrollContainer.scrollLeft; + const beforeY = scrollContainer.scrollTop; + scrollContainer.scrollBy({ left: x, top: y, behavior: 'auto' }); + dispatched = true; + await settleMutationDelivery(); + verified = scrollContainer.scrollLeft !== beforeX || scrollContainer.scrollTop !== beforeY || (x === 0 && y === 0); + break; + } + case 'scroll_into_view': { + strategy = 'DOM_SCROLL'; + const element = this.resolveTarget(documentId, command.target); + const alignment = requireString(command.alignment, 'command.alignment').toLowerCase(); + const block = alignment === 'start' || alignment === 'end' || alignment === 'center' + ? alignment + : 'nearest'; + scrollElementIntoView(element, block); + dispatched = true; + await settleMutationDelivery(); + verified = intersectsViewport( + element.getBoundingClientRect(), + element.ownerDocument.defaultView ?? window, + ); + break; + } + case 'press_keys': { + strategy = 'DOM_KEYBOARD'; + const chord = requireRecord(command.chord, 'command.chord'); + const key = requireString(chord.key, 'command.chord.key'); + const target = deepActiveElement(document) ?? document.body; + if (!target) throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'No active keyboard target is available'); + const before = verificationSnapshot(target, this.revisions); + let keyDownObserved = false; + let keyUpObserved = false; + target.addEventListener('keydown', () => { keyDownObserved = true; }, { once: true, capture: true }); + target.addEventListener('keyup', () => { keyUpObserved = true; }, { once: true, capture: true }); + dispatchKey(target, 'keydown', key, chord); + dispatchKey(target, 'keyup', key, chord); + dispatched = true; + await settleMutationDelivery(); + verified = keyDownObserved && keyUpObserved && hasObservableEffect(before, target, this.revisions); + break; + } + default: + throw new RuntimeHandlerError('UNSUPPORTED_ACTION', `Unsupported command type: ${type}`, { type }); + } + + return { + commandType: type, + strategy, + documentId, + revisionBefore, + revisionAfter: this.revisions.current, + dispatched, + verified, + pageChanged: location.href !== urlBefore, + }; + } + + public async prepareNativeClick( + documentId: string, + candidate: unknown, + frameOffset: (frameId: string) => { left: number; top: number } | null, + requestedOptions: Partial = {}, + ): Promise> { + if (documentId !== this.activeDocumentId()) { + throw new RuntimeHandlerError('STALE_DOCUMENT', 'Command belongs to an inactive document'); + } + const target = parseElementReference(candidate); + const element = this.resolveTarget(documentId, candidate); + requireEnabled(element); + scrollElementIntoView(element, 'center'); + await waitForStableGeometry(element, actionOptions(requestedOptions)); + requireNotOccluded(element); + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) { + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Element has no actionable geometry'); + } + const offset = frameOffset(target.frameId); + if (!offset) throw new RuntimeHandlerError('UNSUPPORTED_FRAME', 'Frame geometry is unavailable'); + const token = `native-click-${this.nextNativePointerToken++}`; + const preparation = { + element, + listener: (() => undefined) as EventListener, + observed: false, + before: verificationSnapshot(element, this.revisions), + timeoutId: 0, + }; + preparation.listener = () => { preparation.observed = true; }; + element.addEventListener('click', preparation.listener, { once: true, capture: true }); + this.nativePointerPreparations.set(token, preparation); + preparation.timeoutId = window.setTimeout(() => { + const expired = this.nativePointerPreparations.get(token); + if (!expired) return; + expired.element.removeEventListener('click', expired.listener, true); + this.nativePointerPreparations.delete(token); + }, 10_000); + return { + token, + xCssPx: offset.left + rect.left + rect.width / 2, + yCssPx: offset.top + rect.top + rect.height / 2, + viewportWidthCssPx: window.innerWidth, + viewportHeightCssPx: window.innerHeight, + revisionBefore: preparation.before.revision, + }; + } + + public verifyNativeClick(documentId: string, token: string): CommandReceipt { + if (documentId !== this.activeDocumentId()) { + throw new RuntimeHandlerError('STALE_DOCUMENT', 'Command belongs to an inactive document'); + } + const preparation = this.nativePointerPreparations.get(token); + if (!preparation) throw new RuntimeHandlerError('INVALID_ACTION', 'Native pointer token is unknown or expired'); + this.nativePointerPreparations.delete(token); + window.clearTimeout(preparation.timeoutId); + preparation.element.removeEventListener('click', preparation.listener, true); + return { + commandType: 'click', + strategy: 'ANDROID_NATIVE_POINTER', + documentId, + revisionBefore: preparation.before.revision, + revisionAfter: this.revisions.current, + dispatched: true, + verified: preparation.observed && hasObservableEffect(preparation.before, preparation.element, this.revisions), + pageChanged: location.href !== preparation.before.url, + }; + } + + public reset(): void { + for (const preparation of this.nativePointerPreparations.values()) { + window.clearTimeout(preparation.timeoutId); + preparation.element.removeEventListener('click', preparation.listener, true); + } + this.nativePointerPreparations.clear(); + } + + private resolveTarget(documentId: string, candidate: unknown): Element { + const target = parseElementReference(candidate); + if (!this.validatesReference(documentId, target.frameId, target.documentId)) { + throw new RuntimeHandlerError('STALE_ELEMENT', 'Element reference belongs to another document or frame'); + } + const element = this.registry.resolve(target.elementId); + if (!element) { + throw new RuntimeHandlerError('STALE_ELEMENT', 'Element is detached or no longer registered', { + elementId: target.elementId, + }); + } + if (target.observedAtRevision > this.revisions.current) { + throw new RuntimeHandlerError('STALE_ELEMENT', 'Element reference has an invalid future revision', { + observedAtRevision: target.observedAtRevision, + currentRevision: this.revisions.current, + }); + } + return element; + } +} + +interface VerificationSnapshot { + revision: number; + url: string; + activeElement: Element | null; + connected: boolean; + checked: boolean | undefined; + selected: boolean | undefined; + value: string | undefined; + expanded: string | null; + pressed: string | null; + text: string; +} + +function verificationSnapshot(element: Element, revisions: DocumentRevisionTracker): VerificationSnapshot { + const stateful = element as Element & { checked?: boolean; selected?: boolean; value?: string }; + return { + revision: revisions.current, + url: location.href, + activeElement: deepActiveElement(document), + connected: element.isConnected, + checked: typeof stateful.checked === 'boolean' ? stateful.checked : undefined, + selected: typeof stateful.selected === 'boolean' ? stateful.selected : undefined, + value: typeof stateful.value === 'string' ? stateful.value : undefined, + expanded: element.getAttribute('aria-expanded'), + pressed: element.getAttribute('aria-pressed'), + text: element.textContent || '', + }; +} + +function hasObservableEffect( + before: VerificationSnapshot, + element: Element, + revisions: DocumentRevisionTracker, +): boolean { + const after = verificationSnapshot(element, revisions); + return after.revision !== before.revision || + after.url !== before.url || + after.activeElement !== before.activeElement || + after.connected !== before.connected || + after.checked !== before.checked || + after.selected !== before.selected || + after.value !== before.value || + after.expanded !== before.expanded || + after.pressed !== before.pressed || + after.text !== before.text; +} + +function parseElementReference(candidate: unknown): ElementReferencePayload { + const value = requireRecord(candidate, 'target'); + const observedAtRevision = requireFiniteNumber(value.observedAtRevision, 'target.observedAtRevision'); + if (!Number.isInteger(observedAtRevision) || observedAtRevision < 0) { + throw new RuntimeHandlerError('INVALID_ACTION', 'target.observedAtRevision must be a non-negative integer'); + } + return { + documentId: requireString(value.documentId, 'target.documentId'), + frameId: requireString(value.frameId, 'target.frameId'), + elementId: requireString(value.elementId, 'target.elementId'), + observedAtRevision, + }; +} + +function requireEnabled(element: Element): void { + const control = element as HTMLElement & { disabled?: boolean; readOnly?: boolean }; + if (control.disabled || element.getAttribute('aria-disabled') === 'true') { + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Element is disabled'); + } + if (control.readOnly || element.hasAttribute('readonly')) { + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Element is read-only'); + } +} + +function setText(element: Element, text: string, mode: string): string { + const current = currentTextValue(element); + const next = mode === 'CLEAR' ? '' + : mode === 'APPEND' ? current + text + : mode === 'INSERT_AT_SELECTION' ? insertAtSelection(element, text) + : mode === 'REPLACE_ALL' ? text + : (() => { throw new RuntimeHandlerError('INVALID_ACTION', `Unsupported text input mode: ${mode}`); })(); + + const tagName = element.tagName.toLowerCase(); + if (tagName === 'input' || tagName === 'textarea') { + const view = element.ownerDocument.defaultView ?? window; + const prototype = tagName === 'input' ? view.HTMLInputElement.prototype : view.HTMLTextAreaElement.prototype; + const setter = Object.getOwnPropertyDescriptor(prototype, 'value')?.set; + if (!setter) throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Native value setter is unavailable'); + setter.call(element, next); + element.dispatchEvent(new view.InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); + element.dispatchEvent(new view.Event('change', { bubbles: true })); + return next; + } + const editable = element as HTMLElement; + if (editable.isContentEditable) { + editable.textContent = next; + const view = element.ownerDocument.defaultView ?? window; + editable.dispatchEvent(new view.InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); + return next; + } + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Target does not accept text input'); +} + +function currentTextValue(element: Element): string { + const tagName = element.tagName.toLowerCase(); + if (tagName === 'input' || tagName === 'textarea') return (element as HTMLInputElement).value; + if ((element as HTMLElement).isContentEditable) return element.textContent || ''; + return ''; +} + +function requireNotOccluded(element: Element): void { + const ownerDocument = element.ownerDocument; + const view = ownerDocument.defaultView ?? window; + if (typeof ownerDocument.elementFromPoint !== 'function') return; + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) { + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Element has no actionable geometry'); + } + const x = Math.min(Math.max(rect.left + rect.width / 2, 0), Math.max(0, view.innerWidth - 1)); + const y = Math.min(Math.max(rect.top + rect.height / 2, 0), Math.max(0, view.innerHeight - 1)); + const top = ownerDocument.elementFromPoint(x, y); + if (top && top !== element && !element.contains(top) && !top.contains(element)) { + throw new RuntimeHandlerError('ELEMENT_OCCLUDED', 'Another element covers the target center point'); + } +} + +function insertAtSelection(element: Element, text: string): string { + const tagName = element.tagName.toLowerCase(); + if (tagName === 'input' || tagName === 'textarea') { + const input = element as HTMLInputElement; + const value = input.value; + const start = input.selectionStart ?? value.length; + const end = input.selectionEnd ?? start; + return value.slice(0, start) + text + value.slice(end); + } + return currentTextValue(element) + text; +} + +function selectOption(select: HTMLSelectElement, matcher: Record): HTMLOptionElement { + const type = requireString(matcher.type, 'command.option.type'); + let found: HTMLOptionElement | undefined; + if (type === 'value') { + const value = requireString(matcher.value, 'command.option.value', true); + found = Array.from(select.options).find(option => option.value === value); + } else if (type === 'label') { + const label = requireString(matcher.label, 'command.option.label', true); + found = Array.from(select.options).find(option => (option.label || option.textContent || '').trim() === label); + } else if (type === 'index') { + const index = requireFiniteNumber(matcher.index, 'command.option.index'); + if (!Number.isInteger(index) || index < 0) throw new RuntimeHandlerError('INVALID_ACTION', 'Option index must be non-negative'); + found = select.options.item(index) || undefined; + } else { + throw new RuntimeHandlerError('INVALID_ACTION', `Unsupported option matcher: ${type}`); + } + if (!found) throw new RuntimeHandlerError('OPTION_NOT_FOUND', 'No matching select option was found'); + if (found.disabled) throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Matching select option is disabled'); + return found; +} + +function nearestScrollable(element: Element): Element { + let current: Element | null = element; + const ownerDocument = element.ownerDocument; + const view = ownerDocument.defaultView ?? window; + while (current && current !== ownerDocument.documentElement) { + const style = view.getComputedStyle(current); + if ((style.overflowY === 'auto' || style.overflowY === 'scroll') && current.scrollHeight > current.clientHeight) { + return current; + } + current = current.parentElement; + } + return ownerDocument.scrollingElement || ownerDocument.documentElement; +} + +function scrollElementIntoView(element: Element, block: ScrollLogicalPosition): void { + const scrollable = element as Element & { scrollIntoView?: (options: ScrollIntoViewOptions) => void }; + scrollable.scrollIntoView?.({ block, inline: 'nearest', behavior: 'auto' }); +} + +function intersectsViewport(rect: DOMRect, view: Window): boolean { + return rect.bottom >= 0 && rect.right >= 0 && rect.top <= view.innerHeight && rect.left <= view.innerWidth; +} + +function pointerButton(value: unknown): number { + const normalized = value === undefined ? 'PRIMARY' : requireString(value, 'command.button'); + if (normalized === 'PRIMARY') return 0; + if (normalized === 'MIDDLE') return 1; + if (normalized === 'SECONDARY') return 2; + throw new RuntimeHandlerError('INVALID_ACTION', `Unsupported pointer button: ${normalized}`); +} + +function dispatchPointerSequence(element: Element, button: number, longPress: boolean): void { + dispatchPointerDown(element, button); + dispatchPointerUp(element, button); + const view = element.ownerDocument.defaultView ?? window; + if (button === 0 && !longPress && typeof (element as HTMLElement).click === 'function') { + (element as HTMLElement).click(); + } else { + const type = button === 2 ? 'contextmenu' : 'click'; + element.dispatchEvent(new view.MouseEvent(type, { + bubbles: true, + cancelable: true, + view, + button, + })); + } +} + +function dispatchPointerDown(element: Element, button: number): void { + dispatchPointerEvent(element, 'pointerdown', button); + dispatchPointerEvent(element, 'mousedown', button); +} + +function dispatchPointerUp(element: Element, button: number): void { + dispatchPointerEvent(element, 'pointerup', button); + dispatchPointerEvent(element, 'mouseup', button); +} + +function dispatchPointerEvent(element: Element, type: string, button: number): void { + const view = element.ownerDocument.defaultView ?? window; + const rect = element.getBoundingClientRect(); + const init: MouseEventInit = { + bubbles: true, + cancelable: true, + view, + button, + clientX: rect.left + rect.width / 2, + clientY: rect.top + rect.height / 2, + }; + const Pointer = view.PointerEvent; + element.dispatchEvent(Pointer ? new Pointer(type, init) : new view.MouseEvent(type, init)); +} + +function deepActiveElement(root: Document | ShadowRoot): Element | null { + const active = root.activeElement; + if (!active) return null; + if (active.shadowRoot?.activeElement) return deepActiveElement(active.shadowRoot); + if (active.tagName.toLowerCase() === 'iframe') { + try { + const childDocument = (active as HTMLIFrameElement).contentDocument; + if (childDocument?.activeElement) return deepActiveElement(childDocument); + } catch { + return active; + } + } + return active; +} + +function dispatchKey( + target: Element, + type: 'keydown' | 'keyup', + key: string, + chord: Record, +): void { + const view = target.ownerDocument.defaultView ?? window; + target.dispatchEvent(new view.KeyboardEvent(type, { + key, + code: key.length === 1 ? `Key${key.toUpperCase()}` : key, + ctrlKey: chord.control === true, + altKey: chord.alt === true, + shiftKey: chord.shift === true, + metaKey: chord.meta === true, + bubbles: true, + cancelable: true, + })); +} + +function delay(durationMs: number): Promise { + return new Promise(resolve => setTimeout(resolve, durationMs)); +} + +function settleMutationDelivery(): Promise { + return new Promise(resolve => setTimeout(resolve, 0)); +} + +async function waitForStableGeometry(element: Element, options: ActionExecutionOptions): Promise { + let previous = element.getBoundingClientRect(); + let stableCycles = 0; + for (let attempt = 0; attempt < options.geometryStableCycles * 4; attempt++) { + await settleMutationDelivery(); + const current = element.getBoundingClientRect(); + if (rectDistance(previous, current) <= options.geometryTolerancePx) { + stableCycles++; + if (stableCycles >= options.geometryStableCycles) return; + } else { + stableCycles = 0; + } + previous = current; + } + throw new RuntimeHandlerError('ELEMENT_NOT_ACTIONABLE', 'Element geometry did not stabilize'); +} + +function rectDistance(left: DOMRect, right: DOMRect): number { + return Math.max( + Math.abs(left.left - right.left), + Math.abs(left.top - right.top), + Math.abs(left.width - right.width), + Math.abs(left.height - right.height), + ); +} + +function actionOptions(candidate: Partial): ActionExecutionOptions { + const options = { ...DEFAULT_ACTION_OPTIONS, ...candidate }; + if (!Number.isInteger(options.geometryStableCycles) || options.geometryStableCycles < 1 || options.geometryStableCycles > 20) { + throw new RuntimeHandlerError('INVALID_ACTION', 'geometryStableCycles must be an integer within 1..20'); + } + if (!Number.isFinite(options.geometryTolerancePx) || options.geometryTolerancePx < 0 || options.geometryTolerancePx > 100) { + throw new RuntimeHandlerError('INVALID_ACTION', 'geometryTolerancePx must be within 0..100'); + } + return options; +} + +function requireRecord(value: unknown, field: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new RuntimeHandlerError('INVALID_ACTION', `${field} must be an object`); + } + return value as Record; +} + +function requireString(value: unknown, field: string, allowEmpty: boolean = false): string { + if (typeof value !== 'string' || (!allowEmpty && value.trim().length === 0)) { + throw new RuntimeHandlerError('INVALID_ACTION', `${field} must be a ${allowEmpty ? '' : 'non-blank '}string`); + } + return value; +} + +function requireFiniteNumber(value: unknown, field: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new RuntimeHandlerError('INVALID_ACTION', `${field} must be a finite number`); + } + return value; +} diff --git a/web-runtime/src/runtime/elementRegistry.test.ts b/web-runtime/src/runtime/elementRegistry.test.ts new file mode 100644 index 0000000..9dbe9c6 --- /dev/null +++ b/web-runtime/src/runtime/elementRegistry.test.ts @@ -0,0 +1,42 @@ +import { ElementRegistry } from './elementRegistry'; + +describe('ElementRegistry', () => { + beforeEach(() => { + document.body.innerHTML = ''; + }); + + it('keeps the same identity when unrelated siblings are inserted', () => { + const registry = new ElementRegistry(); + const button = document.createElement('button'); + document.body.appendChild(button); + const original = registry.getOrCreate(button); + + document.body.insertBefore(document.createElement('div'), button); + + expect(registry.getOrCreate(button)).toBe(original); + }); + + it('assigns a new identity to a replacement element', () => { + const registry = new ElementRegistry(); + const original = document.createElement('button'); + document.body.appendChild(original); + const originalId = registry.getOrCreate(original); + const replacement = document.createElement('button'); + + original.replaceWith(replacement); + + expect(registry.getOrCreate(replacement)).not.toBe(originalId); + expect(registry.resolve(originalId)).toBeUndefined(); + }); + + it('prunes disconnected elements', () => { + const registry = new ElementRegistry(); + const button = document.createElement('button'); + document.body.appendChild(button); + registry.getOrCreate(button); + button.remove(); + + expect(registry.prune()).toBe(1); + expect(registry.size).toBe(0); + }); +}); diff --git a/web-runtime/src/runtime/elementRegistry.ts b/web-runtime/src/runtime/elementRegistry.ts new file mode 100644 index 0000000..1f89397 --- /dev/null +++ b/web-runtime/src/runtime/elementRegistry.ts @@ -0,0 +1,52 @@ +export class ElementRegistry { + private elementToId = new WeakMap(); + private idToElement = new Map(); + private nextIdByFrame = new Map(); + + constructor(private readonly frameId: string = 'main') { + if (!frameId.trim()) throw new Error('frameId must not be blank'); + } + + public getOrCreate(element: Element, frameId: string = this.frameId): string { + const existing = this.elementToId.get(element); + if (existing) return existing; + + const sequence = this.nextIdByFrame.get(frameId) ?? 1; + const id = `${frameId}:e${sequence}`; + this.nextIdByFrame.set(frameId, sequence + 1); + this.elementToId.set(element, id); + this.idToElement.set(id, element); + return id; + } + + public resolve(id: string): Element | undefined { + const element = this.idToElement.get(id); + if (!element) return undefined; + if (!element.isConnected) { + this.idToElement.delete(id); + return undefined; + } + return element; + } + + public prune(): number { + let removed = 0; + for (const [id, element] of this.idToElement) { + if (!element.isConnected) { + this.idToElement.delete(id); + removed++; + } + } + return removed; + } + + public reset(): void { + this.elementToId = new WeakMap(); + this.idToElement.clear(); + this.nextIdByFrame.clear(); + } + + public get size(): number { + return this.idToElement.size; + } +} diff --git a/web-runtime/src/runtime/experimentalPagePatches.ts b/web-runtime/src/runtime/experimentalPagePatches.ts new file mode 100644 index 0000000..28c12d9 --- /dev/null +++ b/web-runtime/src/runtime/experimentalPagePatches.ts @@ -0,0 +1,40 @@ +export interface ExperimentalPagePatchConfiguration { + hideWebDriverProperty?: boolean; + installChromeLikeGlobals?: boolean; + forceFutureShadowRootsOpen?: boolean; +} + +let webdriverPatched = false; +let chromeGlobalsInstalled = false; +let shadowAttachmentPatched = false; + +export function installExperimentalPagePatches(config: ExperimentalPagePatchConfiguration): void { + if (config.hideWebDriverProperty === true && !webdriverPatched) { + const descriptor = Object.getOwnPropertyDescriptor(Navigator.prototype, 'webdriver'); + if (!descriptor || descriptor.configurable) { + Object.defineProperty(Navigator.prototype, 'webdriver', { + configurable: true, + get: () => undefined, + }); + webdriverPatched = true; + } + } + + if (config.installChromeLikeGlobals === true && !chromeGlobalsInstalled) { + const target = window as Window & { chrome?: Record }; + if (!target.chrome) target.chrome = Object.freeze({ runtime: Object.freeze({}) }); + chromeGlobalsInstalled = true; + } + + if (config.forceFutureShadowRootsOpen === true && !shadowAttachmentPatched) { + const original = Element.prototype.attachShadow; + Object.defineProperty(Element.prototype, 'attachShadow', { + configurable: true, + writable: true, + value(init: ShadowRootInit): ShadowRoot { + return original.call(this, { ...init, mode: 'open' }); + }, + }); + shadowAttachmentPatched = true; + } +} diff --git a/web-runtime/src/runtime/frameRegistry.test.ts b/web-runtime/src/runtime/frameRegistry.test.ts new file mode 100644 index 0000000..9839d17 --- /dev/null +++ b/web-runtime/src/runtime/frameRegistry.test.ts @@ -0,0 +1,71 @@ +import { FrameRegistry } from './frameRegistry'; + +describe('FrameRegistry', () => { + beforeEach(() => { + document.body.innerHTML = ''; + }); + + it('keeps frame IDs stable and assigns distinct per-frame document IDs', () => { + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + iframe.contentDocument!.body.innerHTML = ''; + const registry = new FrameRegistry(); + + const first = registry.snapshot('document-1', 8); + const second = registry.snapshot('document-1', 8); + + expect(first).toHaveLength(2); + expect(first[1].id).toBe(second[1].id); + expect(first[1].documentId).toBe(second[1].documentId); + expect(first[1].documentId).toMatch(new RegExp(`^document-1#${first[1].id}:d\\d+$`)); + expect(first[1].capability).toBe('OBSERVABLE_AND_ACTIONABLE'); + }); + + it('reports sandboxed frames without traversing them', () => { + const iframe = document.createElement('iframe'); + iframe.setAttribute('sandbox', 'allow-scripts'); + document.body.appendChild(iframe); + const registry = new FrameRegistry(); + + const frames = registry.snapshot('document-1', 8); + + expect(frames[1].capability).toBe('SANDBOX_RESTRICTED'); + expect(frames[1].document).toBeNull(); + }); + + it('uses collision-free IDs for nested frames', () => { + const outer = document.createElement('iframe'); + document.body.appendChild(outer); + const inner = outer.contentDocument!.createElement('iframe'); + outer.contentDocument!.body.appendChild(inner); + const registry = new FrameRegistry(); + + const frames = registry.snapshot('document-1', 8); + + expect(frames.map(frame => frame.id)).toEqual(['main', 'main:f1', 'main:f2']); + expect(frames[2].parentId).toBe('main:f1'); + expect(frames[2].depth).toBe(2); + }); + + it('reports when nested frames exceed the configured depth', () => { + const outer = document.createElement('iframe'); + document.body.appendChild(outer); + outer.contentDocument!.body.appendChild(outer.contentDocument!.createElement('iframe')); + const registry = new FrameRegistry(); + + const frames = registry.snapshot('document-1', 1); + + expect(frames).toHaveLength(2); + expect(registry.didReachDepthLimit()).toBe(true); + }); + + it('bounds frame discovery work', () => { + document.body.innerHTML = '
'; + const registry = new FrameRegistry(); + + const frames = registry.snapshot('document-1', 8, 2); + + expect(frames).toHaveLength(1); + expect(registry.didReachScanLimit()).toBe(true); + }); +}); diff --git a/web-runtime/src/runtime/frameRegistry.ts b/web-runtime/src/runtime/frameRegistry.ts new file mode 100644 index 0000000..3a24250 --- /dev/null +++ b/web-runtime/src/runtime/frameRegistry.ts @@ -0,0 +1,227 @@ +export type RuntimeFrameCapability = + | 'OBSERVABLE_AND_ACTIONABLE' + | 'INACCESSIBLE_CROSS_ORIGIN' + | 'SANDBOX_RESTRICTED'; + +export interface RuntimeFrame { + id: string; + parentId: string | null; + documentId: string; + document: Document | null; + window: Window | null; + hostElement: HTMLIFrameElement | null; + depth: number; + offsetLeftCssPx: number; + offsetTopCssPx: number; + url: string | null; + origin: string | null; + capability: RuntimeFrameCapability; +} + +export class FrameRegistry { + private elementIds = new WeakMap(); + private documentIds = new WeakMap(); + private nextId = 1; + private nextDocumentGeneration = 1; + private rootDocumentId = ''; + private latestFrames = new Map(); + private depthLimitReached = false; + private scanLimitReached = false; + private remainingScanBudget = 0; + + public snapshot( + rootDocumentId: string, + maximumDepth: number, + maximumScannedElements: number = 10_000, + ): RuntimeFrame[] { + if (!Number.isInteger(maximumDepth) || maximumDepth < 0) { + throw new Error('maximumFrameDepth must be a non-negative integer'); + } + if (!Number.isInteger(maximumScannedElements) || maximumScannedElements < 1) { + throw new Error('maximumScannedElements must be a positive integer'); + } + const frames: RuntimeFrame[] = []; + this.rootDocumentId = rootDocumentId; + this.depthLimitReached = false; + this.scanLimitReached = false; + this.remainingScanBudget = maximumScannedElements; + const mainWindow = document.defaultView ?? window; + const main: RuntimeFrame = { + id: 'main', + parentId: null, + documentId: rootDocumentId, + document, + window: mainWindow, + hostElement: null, + depth: 0, + offsetLeftCssPx: 0, + offsetTopCssPx: 0, + url: safeUrl(mainWindow), + origin: safeOrigin(mainWindow), + capability: 'OBSERVABLE_AND_ACTIONABLE', + }; + frames.push(main); + this.discoverChildren(main, maximumDepth, frames); + this.latestFrames = new Map(frames.map(frame => [frame.id, frame])); + return frames; + } + + public validatesReference(rootDocumentId: string, frameId: string, documentId: string): boolean { + const frame = this.latestFrames.get(frameId); + return frame?.capability === 'OBSERVABLE_AND_ACTIONABLE' && + frame.documentId === documentId && + (frameId !== 'main' || documentId === rootDocumentId); + } + + public frame(frameId: string): RuntimeFrame | undefined { + return this.latestFrames.get(frameId); + } + + public didReachDepthLimit(): boolean { + return this.depthLimitReached; + } + + public didReachScanLimit(): boolean { + return this.scanLimitReached; + } + + public reset(): void { + this.elementIds = new WeakMap(); + this.documentIds = new WeakMap(); + this.nextId = 1; + this.nextDocumentGeneration = 1; + this.rootDocumentId = ''; + this.scanLimitReached = false; + this.remainingScanBudget = 0; + this.latestFrames.clear(); + } + + private discoverChildren(parent: RuntimeFrame, maximumDepth: number, output: RuntimeFrame[]): void { + if (!parent.document) return; + if (parent.depth >= maximumDepth) { + if (this.discoverIframes(parent.document).length > 0) this.depthLimitReached = true; + return; + } + for (const iframe of this.discoverIframes(parent.document)) { + const id = this.idFor(iframe); + const frame = this.describeFrame(iframe, id, parent); + output.push(frame); + if (frame.capability === 'OBSERVABLE_AND_ACTIONABLE') { + this.discoverChildren(frame, maximumDepth, output); + } + } + } + + private describeFrame(iframe: HTMLIFrameElement, id: string, parent: RuntimeFrame): RuntimeFrame { + const sandboxTokens = (iframe.getAttribute('sandbox') || '') + .split(/\s+/) + .filter(Boolean); + const sandboxRestricted = iframe.hasAttribute('sandbox') && + !sandboxTokens.includes('allow-same-origin'); + const rect = iframe.getBoundingClientRect(); + const base = { + id, + parentId: parent.id, + hostElement: iframe, + depth: parent.depth + 1, + offsetLeftCssPx: parent.offsetLeftCssPx + finite(rect.left) + iframe.clientLeft, + offsetTopCssPx: parent.offsetTopCssPx + finite(rect.top) + iframe.clientTop, + }; + if (sandboxRestricted) { + return { + ...base, + documentId: this.unavailableDocumentId(id), + document: null, + window: null, + url: safeAttributeUrl(iframe), + origin: null, + capability: 'SANDBOX_RESTRICTED', + }; + } + try { + const childDocument = iframe.contentDocument; + const childWindow = iframe.contentWindow; + if (!childDocument || !childWindow) throw new Error('Frame document is unavailable'); + void childDocument.documentElement; + return { + ...base, + documentId: this.idForDocument(childDocument, id), + document: childDocument, + window: childWindow, + url: safeUrl(childWindow), + origin: safeOrigin(childWindow), + capability: 'OBSERVABLE_AND_ACTIONABLE', + }; + } catch { + return { + ...base, + documentId: this.unavailableDocumentId(id), + document: null, + window: null, + url: safeAttributeUrl(iframe), + origin: null, + capability: 'INACCESSIBLE_CROSS_ORIGIN', + }; + } + } + + private idFor(iframe: HTMLIFrameElement): string { + const existing = this.elementIds.get(iframe); + if (existing) return existing; + const id = `main:f${this.nextId++}`; + this.elementIds.set(iframe, id); + return id; + } + + private idForDocument(target: Document, frameId: string): string { + const existing = this.documentIds.get(target); + if (existing) return existing; + const id = `${this.rootDocumentId}#${frameId}:d${this.nextDocumentGeneration++}`; + this.documentIds.set(target, id); + return id; + } + + private unavailableDocumentId(frameId: string): string { + return `${this.rootDocumentId}#${frameId}:unavailable`; + } + + private discoverIframes(root: Document | ShadowRoot): HTMLIFrameElement[] { + const frames: HTMLIFrameElement[] = []; + const initial = root instanceof Document + ? (root.documentElement ? [root.documentElement] : []) + : Array.from(root.children); + const stack = [...initial].reverse(); + while (stack.length > 0) { + if (this.remainingScanBudget <= 0) { + this.scanLimitReached = true; + break; + } + this.remainingScanBudget--; + const element = stack.pop()!; + if (element.tagName.toLowerCase() === 'iframe') frames.push(element as HTMLIFrameElement); + const descendants = [ + ...Array.from(element.children), + ...(element.shadowRoot ? Array.from(element.shadowRoot.children) : []), + ]; + for (let index = descendants.length - 1; index >= 0; index--) stack.push(descendants[index]); + } + return frames; + } +} + +function safeUrl(target: Window): string | null { + try { return target.location.href || null; } catch { return null; } +} + +function safeOrigin(target: Window): string | null { + try { return target.location.origin || null; } catch { return null; } +} + +function safeAttributeUrl(iframe: HTMLIFrameElement): string | null { + const value = iframe.getAttribute('src'); + return value && value.trim() ? value : null; +} + +function finite(value: number): number { + return Number.isFinite(value) ? value : 0; +} diff --git a/web-runtime/src/runtime/revisionTracker.test.ts b/web-runtime/src/runtime/revisionTracker.test.ts new file mode 100644 index 0000000..8a52033 --- /dev/null +++ b/web-runtime/src/runtime/revisionTracker.test.ts @@ -0,0 +1,29 @@ +import { DocumentRevisionTracker } from './revisionTracker'; + +describe('DocumentRevisionTracker', () => { + it('coalesces a synchronous mutation storm into one observer revision', async () => { + const tracker = new DocumentRevisionTracker(); + tracker.start(document); + + for (let index = 0; index < 1_000; index++) { + document.body.appendChild(document.createElement('span')); + } + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(tracker.current).toBe(1); + tracker.stop(); + }); + + it('tracks mutations across multiple observed frame roots', async () => { + const frameDocument = document.implementation.createHTMLDocument('frame'); + const tracker = new DocumentRevisionTracker(); + tracker.setRoots([document, frameDocument]); + + document.body.appendChild(document.createElement('p')); + frameDocument.body.appendChild(frameDocument.createElement('button')); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(tracker.current).toBe(2); + tracker.stop(); + }); +}); diff --git a/web-runtime/src/runtime/revisionTracker.ts b/web-runtime/src/runtime/revisionTracker.ts new file mode 100644 index 0000000..c8b1f3e --- /dev/null +++ b/web-runtime/src/runtime/revisionTracker.ts @@ -0,0 +1,56 @@ +export class DocumentRevisionTracker { + private revision = 0; + private observers: MutationObserver[] = []; + private listeners = new Set<(revision: number) => void>(); + + public start(root: Node = document): void { + this.stop(); + this.observe(root); + } + + public setRoots(roots: Node[]): void { + this.stop(); + roots.forEach(root => this.observe(root)); + } + + private observe(root: Node): void { + const observer = new MutationObserver(() => this.bump()); + observer.observe(root, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + attributeFilter: [ + 'aria-label', 'aria-labelledby', 'aria-describedby', 'aria-hidden', + 'aria-disabled', 'aria-expanded', 'aria-checked', 'aria-selected', + 'checked', 'class', 'disabled', 'hidden', 'href', 'placeholder', + 'readonly', 'role', 'selected', 'style', 'title', 'type', 'value', + ], + }); + this.observers.push(observer); + } + + public stop(): void { + this.observers.forEach(observer => observer.disconnect()); + this.observers = []; + } + + public reset(): void { + this.revision = 0; + } + + public bump(): number { + this.revision++; + for (const listener of this.listeners) listener(this.revision); + return this.revision; + } + + public subscribe(listener: (revision: number) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + public get current(): number { + return this.revision; + } +} diff --git a/web-runtime/src/runtime/semanticObserver.test.ts b/web-runtime/src/runtime/semanticObserver.test.ts new file mode 100644 index 0000000..ce15373 --- /dev/null +++ b/web-runtime/src/runtime/semanticObserver.test.ts @@ -0,0 +1,178 @@ +import { ElementRegistry } from './elementRegistry'; +import { DocumentRevisionTracker } from './revisionTracker'; +import { SemanticObserver } from './semanticObserver'; + +describe('SemanticObserver', () => { + let registry: ElementRegistry; + let revisions: DocumentRevisionTracker; + let observer: SemanticObserver; + + beforeEach(() => { + document.body.innerHTML = ''; + registry = new ElementRegistry(); + revisions = new DocumentRevisionTracker(); + observer = new SemanticObserver(registry, revisions); + }); + + it('includes readable content as well as actionable controls', () => { + document.body.innerHTML = ` +
+

Account settings

+

Update your public profile.

+ +
+ `; + + const observation = observer.capture('document-1'); + + expect(observation.contentTrust).toBe('UNTRUSTED_WEBPAGE'); + expect(observation.nodes.some(node => node.kind === 'HEADING' && node.text === 'Account settings')).toBe(true); + expect(observation.nodes.some(node => node.kind === 'TEXT' && node.text === 'Update your public profile.')).toBe(true); + const button = observation.nodes.find(node => node.tagName === 'button'); + expect(button?.accessibleName).toBe('Save profile'); + expect(button?.elementRef?.documentId).toBe('document-1'); + }); + + it('preserves an actionable element reference across unrelated mutations', () => { + document.body.innerHTML = '
'; + const first = observer.capture('document-1'); + const firstRef = first.nodes.find(node => node.tagName === 'button')?.elementRef; + + document.querySelector('main')?.prepend(document.createElement('p')); + revisions.bump(); + const second = observer.capture('document-1'); + const secondRef = second.nodes.find(node => node.tagName === 'button')?.elementRef; + + expect(secondRef?.elementId).toBe(firstRef?.elementId); + expect(secondRef?.observedAtRevision).toBe(1); + }); + + it('reports structured truncation when emitted-node budget is reached', () => { + document.body.innerHTML = '

One

Two

Three

'; + + const observation = observer.capture('document-1', { + maximumVisitedNodes: 10, + maximumEmittedNodes: 2, + }); + + expect(observation.nodes).toHaveLength(2); + expect(observation.truncation?.reason).toBe('EMITTED_NODES'); + }); + + it('handles deeply nested hostile markup without recursive call-stack growth', () => { + let parent: Element = document.body; + for (let depth = 0; depth < 2_000; depth++) { + const child = document.createElement('div'); + parent.appendChild(child); + parent = child; + } + + const style = jest.spyOn(window, 'getComputedStyle').mockReturnValue({ + display: 'block', + visibility: 'visible', + opacity: '1', + } as CSSStyleDeclaration); + const observation = (() => { + try { + return observer.capture('document-1', { + maximumVisitedNodes: 1_000, + maximumEmittedNodes: 10, + maximumTraversalMs: 30_000, + }); + } finally { + style.mockRestore(); + } + })(); + + expect(observation.truncation?.reason).toBe('VISITED_NODES'); + }); + + it('does not expose password values', () => { + document.body.innerHTML = ''; + + const observation = observer.capture('document-1'); + const password = observation.nodes.find(node => node.tagName === 'input'); + + expect(JSON.stringify(password)).not.toContain('secret'); + }); + + it('redacts explicitly sensitive page content and accessible metadata', () => { + document.body.innerHTML = '
$42,000
'; + + const observation = observer.capture('document-1'); + const sensitive = observation.nodes.find(node => node.tagName === 'div'); + + expect(sensitive?.text).toBe('[REDACTED]'); + expect(sensitive?.accessibleName).toBe('[REDACTED]'); + expect(JSON.stringify(sensitive)).not.toContain('42,000'); + expect(JSON.stringify(sensitive)).not.toContain('Private balance'); + }); + + it('redacts every descendant of an explicitly sensitive container', () => { + document.body.innerHTML = ` +
+ alpha-bravo-secret +
+ `; + + const observation = observer.capture('document-1'); + const serialized = JSON.stringify(observation.nodes); + + expect(serialized).not.toContain('alpha-bravo-secret'); + expect(serialized).not.toContain('Recovery code'); + }); + + it('removes credentials, query values, and fragments from observed links', () => { + document.body.innerHTML = 'Account'; + + const observation = observer.capture('document-1'); + const link = observation.nodes.find(node => node.tagName === 'a'); + + expect(link?.attributes.href).toContain('example.com/account'); + expect(link?.attributes.href).not.toContain('user'); + expect(link?.attributes.href).not.toContain('pass'); + expect(link?.attributes.href).not.toContain('secret'); + expect(link?.attributes.href).not.toContain('private'); + }); + + it('observes actionable controls inside open Shadow DOM', () => { + const host = document.createElement('div'); + document.body.appendChild(host); + const root = host.attachShadow({ mode: 'open' }); + root.innerHTML = ''; + + const observation = observer.capture('document-1'); + + expect(observation.nodes.some(node => + node.tagName === 'button' && node.accessibleName === 'Shadow action' && node.elementRef, + )).toBe(true); + }); + + it('increments the document revision for mutations inside an observed Shadow root', async () => { + const host = document.createElement('div'); + document.body.appendChild(host); + const root = host.attachShadow({ mode: 'open' }); + root.innerHTML = ''; + observer.capture('document-1'); + + root.querySelector('button')!.textContent = 'After'; + await new Promise(resolve => setTimeout(resolve, 0)); + const observation = observer.capture('document-1'); + + expect(observation.revision).toBe(1); + expect(observation.nodes.some(node => node.accessibleName === 'After')).toBe(true); + }); + + it('reports when the Shadow DOM depth budget is reached', () => { + const outer = document.createElement('div'); + document.body.appendChild(outer); + const inner = document.createElement('div'); + outer.attachShadow({ mode: 'open' }).appendChild(inner); + inner.attachShadow({ mode: 'open' }).innerHTML = ''; + + const observation = observer.capture('document-1', { maximumShadowDepth: 1 }); + + expect(observation.warnings).toContainEqual(expect.objectContaining({ code: 'SHADOW_DEPTH_LIMIT' })); + expect(observation.nodes.some(node => node.accessibleName === 'Too deep')).toBe(false); + }); +}); diff --git a/web-runtime/src/runtime/semanticObserver.ts b/web-runtime/src/runtime/semanticObserver.ts new file mode 100644 index 0000000..b3a30a6 --- /dev/null +++ b/web-runtime/src/runtime/semanticObserver.ts @@ -0,0 +1,607 @@ +import { ElementRegistry } from './elementRegistry'; +import { FrameRegistry, RuntimeFrame } from './frameRegistry'; +import { DocumentRevisionTracker } from './revisionTracker'; + +export interface SemanticObservationOptions { + maximumVisitedNodes: number; + maximumEmittedNodes: number; + maximumTotalTextCharacters: number; + maximumTextCharactersPerNode: number; + maximumTraversalMs: number; + viewportExpansionPx: number; + maximumFrameDepth: number; + maximumShadowDepth: number; + includeCompactText: boolean; +} + +export interface SemanticObservation { + id: string; + capturedAtEpochMs: number; + contentTrust: 'UNTRUSTED_WEBPAGE'; + document: { id: string; url: string; title: string; phase: 'READY' }; + revision: number; + viewport: Record; + frames: SemanticFrame[]; + nodes: SemanticPageNode[]; + compactText: string; + screenshot: null; + truncation: { reason: string; limit: number; observed: number } | null; + warnings: Array<{ code: string; message: string }>; + metrics: { + durationMs: number; + visitedNodeCount: number; + emittedNodeCount: number; + textCharacterCount: number; + encodedByteCount: number; + }; +} + +interface SemanticFrame { + id: string; + parentId: string | null; + documentId: string; + url: string | null; + origin: string | null; + depth: number; + capability: string; +} + +interface SemanticPageNode { + nodeId: string; + parentNodeId: string | null; + frameId: string; + depth: number; + kind: string; + tagName: string | null; + role: string | null; + text: string | null; + accessibleName: string | null; + accessibleDescription: string | null; + attributes: Record; + states: string[]; + bounds: { leftCssPx: number; topCssPx: number; widthCssPx: number; heightCssPx: number } | null; + visibility: string; + elementRef: { + documentId: string; + frameId: string; + elementId: string; + observedAtRevision: number; + } | null; +} + +interface TraversalEntry { + element: Element; + frame: RuntimeFrame; + parentNodeId: string | null; + depth: number; + shadowDepth: number; +} + +const DEFAULT_OPTIONS: SemanticObservationOptions = { + maximumVisitedNodes: 10_000, + maximumEmittedNodes: 750, + maximumTotalTextCharacters: 100_000, + maximumTextCharactersPerNode: 2_000, + maximumTraversalMs: 1_500, + viewportExpansionPx: 0, + maximumFrameDepth: 8, + maximumShadowDepth: 16, + includeCompactText: true, +}; + +const SKIPPED_TAGS = new Set(['script', 'style', 'noscript', 'template', 'meta', 'link']); +const CONTROL_TAGS = new Set(['button', 'input', 'select', 'textarea', 'option', 'summary', 'details']); +const ACTIONABLE_ROLES = new Set([ + 'button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'textbox', 'searchbox', + 'combobox', 'listbox', 'option', 'slider', 'spinbutton', 'menuitem', 'scrollbar', +]); +const PRESERVED_ATTRIBUTES = [ + 'alt', 'autocomplete', 'checked', 'href', 'name', 'placeholder', 'required', + 'role', 'title', 'type', 'aria-checked', 'aria-expanded', 'aria-label', + 'aria-selected', 'aria-describedby', 'aria-labelledby', +]; + +export class SemanticObserver { + private captureSequence = 0; + + constructor( + private readonly registry: ElementRegistry, + private readonly revisions: DocumentRevisionTracker, + private readonly frameRegistry: FrameRegistry = new FrameRegistry(), + ) {} + + public capture( + documentId: string, + requestedOptions: Partial = {}, + ): SemanticObservation { + const options = validateOptions({ ...DEFAULT_OPTIONS, ...requestedOptions }); + const startedAt = performance.now(); + const frames = this.frameRegistry.snapshot( + documentId, + options.maximumFrameDepth, + options.maximumVisitedNodes, + ); + const revisionRoots: Node[] = frames.flatMap(frame => frame.document ? [frame.document] : []); + const revision = this.revisions.current; + const nodes: SemanticPageNode[] = []; + const warnings = frameWarnings(frames); + if (this.frameRegistry.didReachDepthLimit()) { + warnings.push({ + code: 'FRAME_DEPTH_LIMIT', + message: `Frame traversal stopped at depth ${options.maximumFrameDepth}`, + }); + } + if (this.frameRegistry.didReachScanLimit()) { + warnings.push({ + code: 'FRAME_SCAN_LIMIT', + message: `Frame discovery stopped after ${options.maximumVisitedNodes} elements`, + }); + } + let shadowDepthWarningEmitted = false; + let visitedNodeCount = 0; + let textCharacterCount = 0; + let truncation: SemanticObservation['truncation'] = null; + + const setTruncation = (reason: string, limit: number, observed: number): void => { + if (!truncation) truncation = { reason, limit, observed }; + }; + const pending: TraversalEntry[] = []; + + const visit = ( + element: Element, + frame: RuntimeFrame, + parentNodeId: string | null, + depth: number, + shadowDepth: number, + ): void => { + if (truncation) return; + visitedNodeCount++; + if (visitedNodeCount > options.maximumVisitedNodes) { + setTruncation('VISITED_NODES', options.maximumVisitedNodes, visitedNodeCount); + return; + } + const elapsed = performance.now() - startedAt; + if (elapsed > options.maximumTraversalMs) { + setTruncation('TRAVERSAL_TIME', options.maximumTraversalMs, Math.ceil(elapsed)); + return; + } + + const tagName = element.tagName.toLowerCase(); + if (SKIPPED_TAGS.has(tagName) || isHidden(element)) return; + const role = normalizedAttribute(element, 'role'); + const sensitive = isSensitiveElement(element); + const rawText = sensitive ? '[REDACTED]' : meaningfulText(element); + const textBudget = Math.max(0, options.maximumTotalTextCharacters - textCharacterCount); + const text = capText(rawText, Math.min(options.maximumTextCharactersPerNode, textBudget)); + const kind = nodeKind(tagName, role, text); + const isFrameDocumentRoot = element === frame.document?.body; + const shouldEmit = kind !== 'OTHER' || text.length > 0 || isFrameDocumentRoot; + let nextParentId = parentNodeId; + let nextDepth = depth; + + if (shouldEmit) { + if (nodes.length >= options.maximumEmittedNodes) { + setTruncation('EMITTED_NODES', options.maximumEmittedNodes, nodes.length + 1); + return; + } + if (rawText.length > textBudget) { + setTruncation('TOTAL_TEXT', options.maximumTotalTextCharacters, textCharacterCount + rawText.length); + } + const elementId = this.registry.getOrCreate(element, frame.id); + const bounds = elementBounds(element, frame); + const node: SemanticPageNode = { + nodeId: elementId, + parentNodeId, + frameId: frame.id, + depth, + kind: isFrameDocumentRoot ? 'DOCUMENT' : kind, + tagName: tagName || null, + role, + text: text || null, + accessibleName: sensitive ? '[REDACTED]' : accessibleName(element, text) || null, + accessibleDescription: sensitive + ? null + : referencedText(element, element.getAttribute('aria-describedby')) || null, + attributes: safeAttributes(element, sensitive), + states: elementStates(element), + bounds, + visibility: elementVisibility(element, bounds, frame, options.viewportExpansionPx), + elementRef: isActionable(element, tagName, role) ? { + documentId: frame.documentId, + frameId: frame.id, + elementId, + observedAtRevision: revision, + } : null, + }; + nodes.push(node); + textCharacterCount += text.length; + nextParentId = node.nodeId; + nextDepth = depth + 1; + } + + if (element.shadowRoot && !truncation) { + if (shadowDepth >= options.maximumShadowDepth) { + if (!shadowDepthWarningEmitted) { + warnings.push({ + code: 'SHADOW_DEPTH_LIMIT', + message: `Open Shadow DOM traversal stopped at depth ${options.maximumShadowDepth}`, + }); + shadowDepthWarningEmitted = true; + } + } else { + revisionRoots.push(element.shadowRoot); + pushTraversalEntries( + pending, + Array.from(element.shadowRoot.children), + frame, + nextParentId, + nextDepth, + shadowDepth + 1, + ); + } + } + pushTraversalEntries( + pending, + Array.from(element.children), + frame, + nextParentId, + nextDepth, + shadowDepth, + ); + }; + + for (const frame of frames) { + if (truncation) break; + if (!frame.document?.body) continue; + const parentNodeId = frame.hostElement && frame.parentId + ? this.registry.getOrCreate(frame.hostElement, frame.parentId) + : null; + pending.push({ + element: frame.document.body, + frame, + parentNodeId, + depth: frame.depth, + shadowDepth: 0, + }); + while (pending.length > 0 && !truncation) { + const entry = pending.pop()!; + visit(entry.element, entry.frame, entry.parentNodeId, entry.depth, entry.shadowDepth); + } + } + + this.revisions.setRoots(revisionRoots); + this.registry.prune(); + const compactText = options.includeCompactText ? serializeCompact(nodes, revision) : ''; + const observation: SemanticObservation = { + id: `observation-${documentId}-${revision}-${++this.captureSequence}`, + capturedAtEpochMs: Date.now(), + contentTrust: 'UNTRUSTED_WEBPAGE', + document: { + id: documentId, + url: safeWindowUrl(window) ?? '', + title: document.title || '', + phase: 'READY', + }, + revision, + viewport: viewport(window, document), + frames: frames.map(toSemanticFrame), + nodes, + compactText, + screenshot: null, + truncation, + warnings, + metrics: { + durationMs: Math.max(0, Math.ceil(performance.now() - startedAt)), + visitedNodeCount, + emittedNodeCount: nodes.length, + textCharacterCount, + encodedByteCount: 0, + }, + }; + observation.metrics.encodedByteCount = utf8ByteLength(JSON.stringify(observation)); + return observation; + } +} + +function validateOptions(options: SemanticObservationOptions): SemanticObservationOptions { + const positiveFields: Array = [ + 'maximumVisitedNodes', 'maximumEmittedNodes', 'maximumTotalTextCharacters', + 'maximumTextCharactersPerNode', 'maximumTraversalMs', + ]; + for (const field of positiveFields) { + const value = options[field]; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1) { + throw new Error(`${field} must be a positive number`); + } + } + if (!Number.isInteger(options.maximumFrameDepth) || options.maximumFrameDepth < 0 || options.maximumFrameDepth > 64) { + throw new Error('maximumFrameDepth must be an integer within 0..64'); + } + if (!Number.isInteger(options.maximumShadowDepth) || options.maximumShadowDepth < 0 || options.maximumShadowDepth > 64) { + throw new Error('maximumShadowDepth must be an integer within 0..64'); + } + if (options.maximumEmittedNodes > options.maximumVisitedNodes) { + throw new Error('maximumEmittedNodes cannot exceed maximumVisitedNodes'); + } + if (!Number.isFinite(options.viewportExpansionPx) || options.viewportExpansionPx < -1) { + throw new Error('viewportExpansionPx must be -1 or non-negative'); + } + return options; +} + +function pushTraversalEntries( + pending: TraversalEntry[], + children: Element[], + frame: RuntimeFrame, + parentNodeId: string | null, + depth: number, + shadowDepth: number, +): void { + for (let index = children.length - 1; index >= 0; index--) { + pending.push({ + element: children[index], + frame, + parentNodeId, + depth, + shadowDepth, + }); + } +} + +function toSemanticFrame(frame: RuntimeFrame): SemanticFrame { + return { + id: frame.id, + parentId: frame.parentId, + documentId: frame.documentId, + url: frame.url, + origin: frame.origin, + depth: frame.depth, + capability: frame.capability, + }; +} + +function frameWarnings(frames: RuntimeFrame[]): Array<{ code: string; message: string }> { + return frames + .filter(frame => frame.capability !== 'OBSERVABLE_AND_ACTIONABLE') + .map(frame => ({ + code: frame.capability, + message: `Frame ${frame.id} cannot be observed or acted upon: ${frame.capability}`, + })); +} + +function isHidden(element: Element): boolean { + if (element.hasAttribute('hidden') || element.getAttribute('aria-hidden') === 'true') return true; + try { + const style = element.ownerDocument.defaultView?.getComputedStyle(element); + return style?.display === 'none' || style?.visibility === 'hidden' || + style?.visibility === 'collapse' || style?.opacity === '0'; + } catch { + return false; + } +} + +function meaningfulText(element: Element): string { + const directText = Array.from(element.childNodes) + .filter(node => node.nodeType === 3) + .map(node => node.textContent || '') + .join(' ') + .replace(/\s+/g, ' ') + .trim(); + if (directText) return directText; + return element.children.length === 0 + ? (element.textContent || '').replace(/\s+/g, ' ').trim() + : ''; +} + +function nodeKind(tagName: string, role: string | null, text: string): string { + if (/^h[1-6]$/.test(tagName) || role === 'heading') return 'HEADING'; + if (tagName === 'a' || role === 'link') return 'LINK'; + if (CONTROL_TAGS.has(tagName) || (role && ACTIONABLE_ROLES.has(role))) return 'CONTROL'; + if (['main', 'nav', 'header', 'footer', 'aside', 'section', 'article', 'form'].includes(tagName)) return 'LANDMARK'; + if (tagName === 'ul' || tagName === 'ol' || role === 'list') return 'LIST'; + if (tagName === 'li' || role === 'listitem') return 'LIST_ITEM'; + if (tagName === 'table' || role === 'table') return 'TABLE'; + if (tagName === 'tr' || role === 'row') return 'ROW'; + if (tagName === 'td' || tagName === 'th' || role === 'cell') return 'CELL'; + if (tagName === 'img' || role === 'img') return 'IMAGE'; + if (tagName === 'iframe') return 'FRAME'; + return text ? 'TEXT' : 'OTHER'; +} + +function isActionable(element: Element, tagName: string, role: string | null): boolean { + return (tagName === 'a' && element.hasAttribute('href')) || + CONTROL_TAGS.has(tagName) || + Boolean(role && ACTIONABLE_ROLES.has(role)) || + element.hasAttribute('onclick') || + element.hasAttribute('tabindex') || + (element as HTMLElement).isContentEditable === true; +} + +function accessibleName(element: Element, text: string): string { + return normalizedAttribute(element, 'aria-label') || + referencedText(element, element.getAttribute('aria-labelledby')) || + normalizedAttribute(element, 'alt') || + normalizedAttribute(element, 'title') || + normalizedAttribute(element, 'placeholder') || + text; +} + +function referencedText(element: Element, ids: string | null): string { + if (!ids) return ''; + return ids.split(/\s+/) + .map(id => element.ownerDocument.getElementById(id)?.textContent?.replace(/\s+/g, ' ').trim() || '') + .filter(Boolean) + .join(' '); +} + +function safeAttributes(element: Element, sensitive: boolean): Record { + const result: Record = {}; + const inputType = normalizedAttribute(element, 'type').toLowerCase(); + for (const name of PRESERVED_ATTRIBUTES) { + const value = normalizedAttribute(element, name); + if (!value) continue; + result[name] = sensitive || (inputType === 'password' && name === 'placeholder') + ? '[REDACTED]' + : name === 'href' + ? redactUrlSecrets(value, element.baseURI) + : capText(value, 500); + } + return result; +} + +function isSensitiveElement(element: Element): boolean { + const type = normalizedAttribute(element, 'type').toLowerCase(); + const autocomplete = normalizedAttribute(element, 'autocomplete').toLowerCase(); + return type === 'password' || + hasSensitiveAncestor(element) || + ['current-password', 'new-password', 'cc-number', 'cc-csc', 'one-time-code'].includes(autocomplete); +} + +function hasSensitiveAncestor(element: Element): boolean { + let current: Element | null = element; + while (current) { + if (current.hasAttribute('data-agentic-sensitive')) return true; + const parentElement: Element | null = current.parentElement; + if (parentElement) { + current = parentElement; + continue; + } + const root = current.getRootNode(); + current = root instanceof ShadowRoot ? root.host : null; + } + return false; +} + +function redactUrlSecrets(value: string, baseUrl: string): string { + try { + const url = new URL(value, baseUrl); + url.username = ''; + url.password = ''; + if (url.search) url.search = '?[REDACTED]'; + if (url.hash) url.hash = '#[REDACTED]'; + return capText(url.toString(), 500); + } catch { + return '[REDACTED_URL]'; + } +} + +function elementStates(element: Element): string[] { + const states: string[] = []; + const html = element as HTMLElement & { + disabled?: boolean; readOnly?: boolean; checked?: boolean; selected?: boolean; required?: boolean; + }; + if (html.disabled || element.getAttribute('aria-disabled') === 'true') states.push('DISABLED'); + if (html.readOnly || element.hasAttribute('readonly')) states.push('READ_ONLY'); + if (html.checked || element.getAttribute('aria-checked') === 'true') states.push('CHECKED'); + if (html.selected || element.getAttribute('aria-selected') === 'true') states.push('SELECTED'); + if (html.required || element.hasAttribute('required')) states.push('REQUIRED'); + const expanded = element.getAttribute('aria-expanded'); + if (expanded === 'true') states.push('EXPANDED'); + if (expanded === 'false') states.push('COLLAPSED'); + if (element === element.ownerDocument.activeElement) states.push('FOCUSED'); + if (['input', 'textarea'].includes(element.tagName.toLowerCase()) || html.isContentEditable) states.push('EDITABLE'); + return states; +} + +function elementBounds(element: Element, frame: RuntimeFrame): SemanticPageNode['bounds'] { + try { + const rect = element.getBoundingClientRect(); + return { + leftCssPx: frame.offsetLeftCssPx + finiteOrZero(rect.left), + topCssPx: frame.offsetTopCssPx + finiteOrZero(rect.top), + widthCssPx: Math.max(0, finiteOrZero(rect.width)), + heightCssPx: Math.max(0, finiteOrZero(rect.height)), + }; + } catch { + return null; + } +} + +function elementVisibility( + element: Element, + bounds: SemanticPageNode['bounds'], + frame: RuntimeFrame, + expansion: number, +): string { + if (!bounds || !frame.window || !frame.document) return 'UNKNOWN'; + if (bounds.widthCssPx === 0 && bounds.heightCssPx === 0) return 'UNKNOWN'; + const localLeft = bounds.leftCssPx - frame.offsetLeftCssPx; + const localTop = bounds.topCssPx - frame.offsetTopCssPx; + if (expansion !== -1 && ( + localTop + bounds.heightCssPx < -expansion || + localLeft + bounds.widthCssPx < -expansion || + localTop > frame.window.innerHeight + expansion || + localLeft > frame.window.innerWidth + expansion + )) return 'OFFSCREEN'; + try { + const top = frame.document.elementFromPoint( + localLeft + bounds.widthCssPx / 2, + localTop + bounds.heightCssPx / 2, + ); + if (top && top !== element && !element.contains(top) && !top.contains(element)) return 'OCCLUDED'; + } catch { + return 'UNKNOWN'; + } + return 'VISIBLE'; +} + +function serializeCompact(nodes: SemanticPageNode[], revision: number): string { + const lines = [`[Untrusted webpage observation revision=${revision}]`]; + for (const node of nodes) { + if (node.kind === 'DOCUMENT') { + if (node.frameId !== 'main') lines.push(`[Frame ${node.frameId}]`); + continue; + } + const indent = ' '.repeat(Math.min(node.depth, 12)); + const reference = node.elementRef ? `[${node.frameId}/${node.elementRef.elementId}]` : ''; + const tag = node.tagName || node.kind.toLowerCase(); + const role = node.role ? ` role=${JSON.stringify(node.role)}` : ''; + const name = node.accessibleName ? ` name=${JSON.stringify(capText(node.accessibleName, 200))}` : ''; + const text = node.text && node.text !== node.accessibleName ? ` ${capText(node.text, 500)}` : ''; + lines.push(`${indent}${reference}<${tag}${role}${name}>${text}`.trimEnd()); + } + return lines.join('\n'); +} + +function viewport(targetWindow: Window, targetDocument: Document): Record { + const root = targetDocument.documentElement; + return { + scrollXCssPx: finiteOrZero(targetWindow.scrollX), + scrollYCssPx: finiteOrZero(targetWindow.scrollY), + widthCssPx: finiteOrZero(targetWindow.innerWidth), + heightCssPx: finiteOrZero(targetWindow.innerHeight), + contentWidthCssPx: finiteOrZero(root?.scrollWidth || 0), + contentHeightCssPx: finiteOrZero(root?.scrollHeight || 0), + devicePixelRatio: finiteOrZero(targetWindow.devicePixelRatio || 1), + visualViewportScale: finiteOrZero(targetWindow.visualViewport?.scale || 1), + }; +} + +function safeWindowUrl(target: Window): string | null { + try { return target.location.href || null; } catch { return null; } +} + +function normalizedAttribute(element: Element, name: string): string { + return (element.getAttribute(name) || '').replace(/\s+/g, ' ').trim(); +} + +function capText(value: string, maximum: number): string { + if (maximum <= 0) return ''; + return value.length <= maximum ? value : `${value.slice(0, Math.max(0, maximum - 1))}…`; +} + +function finiteOrZero(value: number): number { + return Number.isFinite(value) ? value : 0; +} + +function utf8ByteLength(value: string): number { + let bytes = 0; + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint <= 0x7f) bytes++; + else if (codePoint <= 0x7ff) bytes += 2; + else if (codePoint <= 0xffff) bytes += 3; + else bytes += 4; + } + return bytes; +} diff --git a/web-runtime/src/runtime/semanticRuntime.ts b/web-runtime/src/runtime/semanticRuntime.ts new file mode 100644 index 0000000..3c1fd86 --- /dev/null +++ b/web-runtime/src/runtime/semanticRuntime.ts @@ -0,0 +1,88 @@ +import { ElementRegistry } from './elementRegistry'; +import { DocumentRevisionTracker } from './revisionTracker'; +import { SemanticObserver, SemanticObservationOptions } from './semanticObserver'; +import { SemanticActionExecutor } from './actionExecutor'; +import { FrameRegistry } from './frameRegistry'; +import { + ExperimentalPagePatchConfiguration, + installExperimentalPagePatches, +} from './experimentalPagePatches'; + +export class SemanticRuntime { + private readonly registry = new ElementRegistry('main'); + private readonly frames = new FrameRegistry(); + private readonly revisions = new DocumentRevisionTracker(); + private readonly observer = new SemanticObserver(this.registry, this.revisions, this.frames); + private readonly actions = new SemanticActionExecutor( + this.registry, + this.revisions, + () => this.activeDocumentId, + (rootDocumentId, frameId, documentId) => + this.frames.validatesReference(rootDocumentId, frameId, documentId), + ); + private activeDocumentId: string | null = null; + + constructor() { + this.revisions.start(document); + } + + public capture(documentId: string, options: Partial = {}) { + this.activate(documentId); + return this.observer.capture(documentId, options); + } + + public configure(payload: Record): void { + const experimental = payload.experimental; + if (experimental === undefined) return; + if (typeof experimental !== 'object' || experimental === null || Array.isArray(experimental)) { + throw new Error('experimental configuration must be an object'); + } + installExperimentalPagePatches(experimental as ExperimentalPagePatchConfiguration); + } + + public capabilities(): Record { + return { + semanticObservation: true, + stableElementReferences: true, + openShadowDom: true, + sameOriginFrames: true, + nativePointerActions: true, + screenshots: true, + }; + } + + public activate(documentId: string): void { + if (!documentId.trim()) throw new Error('documentId must not be blank'); + if (this.activeDocumentId === documentId) return; + this.activeDocumentId = documentId; + this.actions.reset(); + this.registry.reset(); + this.frames.reset(); + this.revisions.reset(); + } + + public resolve(documentId: string, elementId: string): Element | undefined { + if (documentId !== this.activeDocumentId) return undefined; + return this.registry.resolve(elementId); + } + + public execute(documentId: string, command: Record, options: Record = {}) { + return this.actions.execute(documentId, command, options); + } + + public prepareNativeClick(documentId: string, target: unknown, options: Record = {}) { + return this.actions.prepareNativeClick(documentId, target, frameId => { + const frame = this.frames.frame(frameId); + return frame ? { left: frame.offsetLeftCssPx, top: frame.offsetTopCssPx } : null; + }, options); + } + + public verifyNativeClick(documentId: string, token: string) { + return this.actions.verifyNativeClick(documentId, token); + } + + public get revision(): number { + return this.revisions.current; + } + +} diff --git a/web-injector/tsconfig.json b/web-runtime/tsconfig.json similarity index 100% rename from web-injector/tsconfig.json rename to web-runtime/tsconfig.json diff --git a/website/package-lock.json b/website/package-lock.json index 3bc2c11..7739c06 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -1,31 +1,23 @@ { - "name": "react-example", + "name": "agentic-webview-website", "version": "0.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "react-example", + "name": "agentic-webview-website", "version": "0.0.0", "dependencies": { - "@tailwindcss/vite": "^4.1.14", - "@vitejs/plugin-react": "^5.0.4", - "dotenv": "^17.2.3", - "express": "^4.21.2", "lucide-react": "^0.546.0", - "motion": "^12.23.24", "react": "^19.0.1", "react-dom": "^19.0.1", - "react-router-dom": "^7.16.0", - "vite": "^6.2.3" + "react-router-dom": "^7.16.0" }, "devDependencies": { - "@types/express": "^4.17.21", + "@tailwindcss/vite": "^4.1.14", "@types/node": "^22.14.0", - "autoprefixer": "^10.4.21", - "esbuild": "^0.25.0", + "@vitejs/plugin-react": "^5.0.4", "tailwindcss": "^4.1.14", - "tsx": "^4.21.0", "typescript": "~5.8.2", "vite": "^6.2.3" } @@ -34,6 +26,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-validator-identifier": "^7.29.7", @@ -48,6 +41,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -57,6 +51,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.29.7", @@ -87,6 +82,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.29.7", @@ -103,6 +99,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, "license": "MIT", "dependencies": { "@babel/compat-data": "^7.29.7", @@ -119,6 +116,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -128,6 +126,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, "license": "MIT", "dependencies": { "@babel/traverse": "^7.29.7", @@ -141,6 +140,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.29.7", @@ -158,6 +158,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -167,6 +168,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -176,6 +178,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -185,6 +188,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -194,6 +198,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/template": "^7.29.7", @@ -207,6 +212,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.29.7" @@ -222,6 +228,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" @@ -237,6 +244,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" @@ -252,6 +260,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.29.7", @@ -266,6 +275,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.29.7", @@ -284,6 +294,7 @@ "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.29.7", @@ -300,6 +311,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -313,6 +325,7 @@ "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", @@ -323,6 +336,7 @@ "version": "2.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", @@ -333,6 +347,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.0.0" @@ -342,12 +357,14 @@ "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.31", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -358,6 +375,7 @@ "version": "1.0.0-rc.3", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", "integrity": "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==", + "dev": true, "license": "MIT" }, "node_modules/@rollup/rollup-linux-x64-gnu": { @@ -367,6 +385,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -380,6 +399,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -393,6 +413,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -403,6 +424,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.0.tgz", "integrity": "sha512-aFb4gUhFOgdh9AXo4IzBEOzBkkAxm9VigwDJnMIYv3lcfXCJVesNfbEaBl4BNgVRyid92AmdviqwBUBRKSeY3g==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.5", @@ -418,6 +440,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.0.tgz", "integrity": "sha512-F7HZGBeN9I0/AuuJS5PwcD8xayx5ri5GhjYUDBEVYUkexyA/giwbDNjRVrxSezE3T250OU2K/wp/ltWx3UOefg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 20" @@ -444,6 +467,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -460,6 +484,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -476,6 +501,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -485,10 +511,47 @@ "node": ">= 20" } }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-android-arm64": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-darwin-arm64": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-darwin-x64": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-freebsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-wasm32-wasi": { + "dev": true, + "optional": true + }, + "node_modules/@tailwindcss/oxide/node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "dev": true, + "optional": true + }, "node_modules/@tailwindcss/vite": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.0.tgz", "integrity": "sha512-t6J3OrB5Fc0ExuhohouH0fWUGMYL6PTLhW+E7zIk/pdbnJARZDCwjBznFnkh5ynRnIRSI4YjtTH0t6USjJISrw==", + "dev": true, "license": "MIT", "dependencies": { "@tailwindcss/node": "4.3.0", @@ -503,6 +566,7 @@ "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.20.7", @@ -516,6 +580,7 @@ "version": "7.27.0", "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.0.0" @@ -525,6 +590,7 @@ "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, "license": "MIT", "dependencies": { "@babel/parser": "^7.1.0", @@ -535,75 +601,16 @@ "version": "7.28.0", "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.2" - } - }, - "node_modules/@types/body-parser": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", - "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.38", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", - "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", "dev": true, "license": "MIT", "dependencies": { - "@types/node": "*" + "@babel/types": "^7.28.2" } }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "license": "MIT" - }, - "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.19.8", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", - "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*", - "@types/send": "*" - } - }, - "node_modules/@types/http-errors": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", - "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mime": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", - "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true, "license": "MIT" }, @@ -611,63 +618,17 @@ "version": "22.19.19", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.19.tgz", "integrity": "sha512-dyh/xO2Fh5bYrfWaaqGrRQQGkNdmYw6AmaAUvYeUMNTWQtvb796ikLdmTchRmOlOiIJ1TDXfWgVx1QkUlQ6Hew==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@types/qs": { - "version": "6.15.1", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", - "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", - "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", - "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.10", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", - "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "<1" - } - }, - "node_modules/@types/serve-static/node_modules/@types/send": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", - "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", "dev": true, "license": "MIT", "dependencies": { - "@types/mime": "^1", - "@types/node": "*" + "undici-types": "~6.21.0" } }, "node_modules/@vitejs/plugin-react": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", "integrity": "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==", + "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.29.0", @@ -684,66 +645,11 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", - "license": "MIT", - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "license": "MIT" - }, - "node_modules/autoprefixer": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.5.0.tgz", - "integrity": "sha512-FMhOoZV4+qR6aTUALKX2rEqGG+oyATvwBt9IIzVR5rMa2HRWPkxf+P+PAJLD1I/H5/II+HuZcBJYEFBpq39ong==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/autoprefixer" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "browserslist": "^4.28.2", - "caniuse-lite": "^1.0.30001787", - "fraction.js": "^5.3.4", - "picocolors": "^1.1.1", - "postcss-value-parser": "^4.2.0" - }, - "bin": { - "autoprefixer": "bin/autoprefixer" - }, - "engines": { - "node": "^10 || ^12 || >=14" - }, - "peerDependencies": { - "postcss": "^8.1.0" - } - }, "node_modules/baseline-browser-mapping": { "version": "2.10.32", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.32.tgz", "integrity": "sha512-wbPvpyjJPC0zdfdKXxqEL3Ea+bOMD/87X4lftiJkkaBiuG6ALQy1SLmEd7BSmVCuwCQsBrCamgBoLyfFDD1EPg==", + "dev": true, "license": "Apache-2.0", "bin": { "baseline-browser-mapping": "dist/cli.cjs" @@ -752,49 +658,11 @@ "node": ">=6.0.0" } }, - "node_modules/body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/body-parser/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, "node_modules/browserslist": { "version": "4.28.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", + "dev": true, "funding": [ { "type": "opencollective", @@ -824,48 +692,11 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/caniuse-lite": { "version": "1.0.30001793", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001793.tgz", "integrity": "sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==", + "dev": true, "funding": [ { "type": "opencollective", @@ -882,52 +713,18 @@ ], "license": "CC-BY-4.0" }, - "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", - "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "dev": true, "license": "MIT" }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -941,85 +738,28 @@ } } }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, - "node_modules/dotenv": { - "version": "17.4.2", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", - "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, "node_modules/electron-to-chromium": { "version": "1.5.364", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.364.tgz", "integrity": "sha512-G/dYE3+AYhyHwzTwg8UbnXf7zqMERYh7l2jJ3QujhFsH8agSYwtnGAR2aZ7f0AakIKJXd5En/Hre4igIUrdlYw==", + "dev": true, "license": "ISC" }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/enhanced-resolve": { "version": "5.22.1", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.22.1.tgz", "integrity": "sha512-6QEuw3zoX1SJQc7b87aBXke/no+mG2bTBgw29gWMQonLmpEkWoCAVkl+M49e48AZlWzxiDzDZzYdp6kobcyLww==", + "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", @@ -1029,40 +769,11 @@ "node": ">=10.13.0" } }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", - "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/esbuild": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -1100,348 +811,156 @@ "@esbuild/win32-x64": "0.25.12" } }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "license": "MIT", - "engines": { - "node": ">=6" - } + "node_modules/esbuild/node_modules/@esbuild/aix-ppc64": { + "dev": true, + "optional": true }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" + "node_modules/esbuild/node_modules/@esbuild/android-arm": { + "dev": true, + "optional": true }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } + "node_modules/esbuild/node_modules/@esbuild/android-arm64": { + "dev": true, + "optional": true }, - "node_modules/express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } + "node_modules/esbuild/node_modules/@esbuild/android-x64": { + "dev": true, + "optional": true }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } + "node_modules/esbuild/node_modules/@esbuild/darwin-arm64": { + "dev": true, + "optional": true }, - "node_modules/express/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "node_modules/esbuild/node_modules/@esbuild/darwin-x64": { + "dev": true, + "optional": true }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } + "node_modules/esbuild/node_modules/@esbuild/freebsd-arm64": { + "dev": true, + "optional": true }, - "node_modules/finalhandler": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", - "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "statuses": "~2.0.2", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } + "node_modules/esbuild/node_modules/@esbuild/freebsd-x64": { + "dev": true, + "optional": true }, - "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } + "node_modules/esbuild/node_modules/@esbuild/linux-arm": { + "dev": true, + "optional": true }, - "node_modules/finalhandler/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" + "node_modules/esbuild/node_modules/@esbuild/linux-arm64": { + "dev": true, + "optional": true }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } + "node_modules/esbuild/node_modules/@esbuild/linux-ia32": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/linux-loong64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/linux-mips64el": { + "dev": true, + "optional": true }, - "node_modules/fraction.js": { - "version": "5.3.4", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", - "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", + "node_modules/esbuild/node_modules/@esbuild/linux-ppc64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/linux-riscv64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/linux-s390x": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/netbsd-arm64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/netbsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/openbsd-arm64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/openbsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/openharmony-arm64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/sunos-x64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/win32-arm64": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/win32-ia32": { + "dev": true, + "optional": true + }, + "node_modules/esbuild/node_modules/@esbuild/win32-x64": { + "dev": true, + "optional": true + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "dev": true, "license": "MIT", "engines": { - "node": "*" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/rawify" + "node": ">=6" } }, - "node_modules/framer-motion": { - "version": "12.40.0", - "resolved": "https://registry.npmjs.org/framer-motion/-/framer-motion-12.40.0.tgz", - "integrity": "sha512-uaBd3qC1v3KQqBEjwTUd183K6PbS+j0yR9w9VmEOLWA/tnUcSn8Xa3uck7t4dgpDoUss8xQTcj8W2L07lrnLFg==", + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, "license": "MIT", - "dependencies": { - "motion-dom": "^12.40.0", - "motion-utils": "^12.39.0", - "tslib": "^2.4.0" + "engines": { + "node": ">=12.0.0" }, "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" + "picomatch": "^3 || ^4" }, "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { + "picomatch": { "optional": true } } }, - "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" } }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, "license": "ISC" }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/jiti": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, "license": "MIT", "bin": { "jiti": "lib/jiti-cli.mjs" @@ -1451,12 +970,14 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, "license": "MIT" }, "node_modules/jsesc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, "license": "MIT", "bin": { "jsesc": "bin/jsesc" @@ -1469,6 +990,7 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, "license": "MIT", "bin": { "json5": "lib/cli.js" @@ -1481,6 +1003,7 @@ "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, "license": "MPL-2.0", "dependencies": { "detect-libc": "^2.0.3" @@ -1513,6 +1036,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -1533,6 +1057,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -1553,6 +1078,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -1566,10 +1092,43 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/lightningcss/node_modules/lightningcss-android-arm64": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-darwin-arm64": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-darwin-x64": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-freebsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-linux-arm-gnueabihf": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-linux-arm64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-linux-arm64-musl": { + "dev": true, + "optional": true + }, + "node_modules/lightningcss/node_modules/lightningcss-win32-arm64-msvc": { + "dev": true, + "optional": true + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, "license": "ISC", "dependencies": { "yallist": "^3.0.2" @@ -1588,131 +1147,24 @@ "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/motion": { - "version": "12.40.0", - "resolved": "https://registry.npmjs.org/motion/-/motion-12.40.0.tgz", - "integrity": "sha512-yjrHUrBFW6kQvjJwRsoiPSAhC5tRwRqNGJWmiJ4CrGnbKp0V88AdzkhBmDoqIsIPfarOe0Uddd37Xq43/gIocA==", - "license": "MIT", - "dependencies": { - "framer-motion": "^12.40.0", - "tslib": "^2.4.0" - }, - "peerDependencies": { - "@emotion/is-prop-valid": "*", - "react": "^18.0.0 || ^19.0.0", - "react-dom": "^18.0.0 || ^19.0.0" - }, - "peerDependenciesMeta": { - "@emotion/is-prop-valid": { - "optional": true - }, - "react": { - "optional": true - }, - "react-dom": { - "optional": true - } - } - }, - "node_modules/motion-dom": { - "version": "12.40.0", - "resolved": "https://registry.npmjs.org/motion-dom/-/motion-dom-12.40.0.tgz", - "integrity": "sha512-HxU3ZaBwNPVQUBQf1xxgq+7JrPNZvjLVxgbpEZL7RrWJnsxOf0/OM+yrHG9ogLQ31Do/r57Oz2gQWPK+6q62mg==", - "license": "MIT", - "dependencies": { - "motion-utils": "^12.39.0" - } - }, - "node_modules/motion-utils": { - "version": "12.39.0", - "resolved": "https://registry.npmjs.org/motion-utils/-/motion-utils-12.39.0.tgz", - "integrity": "sha512-8nadJAJjTtqRkmRF36FoJTrywK9nnFmnPwnSMyxaOCU7GDjN9RTMJIxx9De8ErM+vpPhMccr/6fo5WciyQLnMQ==", - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, "license": "MIT" }, "node_modules/nanoid": { "version": "3.3.12", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, "funding": [ { "type": "github", @@ -1727,73 +1179,28 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/node-releases": { "version": "2.0.46", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.46.tgz", "integrity": "sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=18" } }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-to-regexp": { - "version": "0.1.13", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", - "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", - "license": "MIT" - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -1806,6 +1213,7 @@ "version": "8.5.15", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, "funding": [ { "type": "opencollective", @@ -1830,65 +1238,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/postcss-value-parser": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.1.0" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/react": { "version": "19.2.6", "resolved": "https://registry.npmjs.org/react/-/react-19.2.6.tgz", @@ -1914,6 +1263,7 @@ "version": "0.18.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -1974,6 +1324,7 @@ "version": "4.60.4", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.4.tgz", "integrity": "sha512-WHeFSbZYsPu3+bLoNRUuAO+wavNlocOPf3wSHTP7hcFKVnJeWsYlCDbr3mTS14FCizf9ccIxXA8sGL8zKeQN3g==", + "dev": true, "license": "MIT", "dependencies": { "@types/estree": "1.0.8" @@ -2014,31 +1365,97 @@ "fsevents": "~2.3.2" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" + "node_modules/rollup/node_modules/@rollup/rollup-android-arm-eabi": { + "dev": true, + "optional": true }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" + "node_modules/rollup/node_modules/@rollup/rollup-android-arm64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-darwin-arm64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-darwin-x64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-freebsd-arm64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-freebsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-arm-musleabihf": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-arm64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-arm64-musl": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-loong64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-loong64-musl": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-ppc64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-ppc64-musl": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-riscv64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-riscv64-musl": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-s390x-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-openbsd-x64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-openharmony-arm64": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-win32-arm64-msvc": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-win32-ia32-msvc": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/@rollup/rollup-win32-x64-gnu": { + "dev": true, + "optional": true + }, + "node_modules/rollup/node_modules/fsevents": { + "dev": true, + "optional": true }, "node_modules/scheduler": { "version": "0.27.0", @@ -2050,177 +1467,40 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" } }, - "node_modules/send": { - "version": "0.19.2", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", - "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.1", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "~2.4.1", - "range-parser": "~1.2.1", - "statuses": "~2.0.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/serve-static": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", - "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", - "license": "MIT", - "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "~0.19.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", "license": "MIT" }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/tailwindcss": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.0.tgz", "integrity": "sha512-y6nxMGB1nMW9R6k96e5gdIFzcfL/gTJRNaqGes1YvkLnPVXzWgbqFF2yLC0T8G774n24cx3Pe8XrKoniCOAH+Q==", + "dev": true, "license": "MIT" }, "node_modules/tapable": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -2234,6 +1514,7 @@ "version": "0.2.16", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", @@ -2246,111 +1527,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/tsx": { - "version": "4.22.3", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.3.tgz", - "integrity": "sha512-mdoNxBC/cSQObGGVQ5Bpn5i+yv7j68gk3Nfm3wFjcJg3Z0Mix9jzAFfP12prmm5eVGmDKtp0yyArrs0Q+8gZHg==", - "devOptional": true, - "license": "MIT", - "dependencies": { - "esbuild": "~0.28.0" - }, - "bin": { - "tsx": "dist/cli.mjs" - }, - "engines": { - "node": ">=18.0.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - } - }, - "node_modules/tsx/node_modules/@esbuild/linux-x64": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", - "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/tsx/node_modules/esbuild": { - "version": "0.28.0", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", - "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==", - "devOptional": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.0", - "@esbuild/android-arm": "0.28.0", - "@esbuild/android-arm64": "0.28.0", - "@esbuild/android-x64": "0.28.0", - "@esbuild/darwin-arm64": "0.28.0", - "@esbuild/darwin-x64": "0.28.0", - "@esbuild/freebsd-arm64": "0.28.0", - "@esbuild/freebsd-x64": "0.28.0", - "@esbuild/linux-arm": "0.28.0", - "@esbuild/linux-arm64": "0.28.0", - "@esbuild/linux-ia32": "0.28.0", - "@esbuild/linux-loong64": "0.28.0", - "@esbuild/linux-mips64el": "0.28.0", - "@esbuild/linux-ppc64": "0.28.0", - "@esbuild/linux-riscv64": "0.28.0", - "@esbuild/linux-s390x": "0.28.0", - "@esbuild/linux-x64": "0.28.0", - "@esbuild/netbsd-arm64": "0.28.0", - "@esbuild/netbsd-x64": "0.28.0", - "@esbuild/openbsd-arm64": "0.28.0", - "@esbuild/openbsd-x64": "0.28.0", - "@esbuild/openharmony-arm64": "0.28.0", - "@esbuild/sunos-x64": "0.28.0", - "@esbuild/win32-arm64": "0.28.0", - "@esbuild/win32-ia32": "0.28.0", - "@esbuild/win32-x64": "0.28.0" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/typescript": { "version": "5.8.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", @@ -2369,22 +1545,14 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "devOptional": true, + "dev": true, "license": "MIT" }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/update-browserslist-db": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, "funding": [ { "type": "opencollective", @@ -2411,28 +1579,11 @@ "browserslist": ">= 4.21.0" } }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/vite": { "version": "6.4.2", "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz", "integrity": "sha512-2N/55r4JDJ4gdrCvGgINMy+HH3iRpNIz8K6SFwVsA+JbQScLiC+clmAxBgwiSPgcG9U15QmvqCGWzMbqda5zGQ==", + "dev": true, "license": "MIT", "dependencies": { "esbuild": "^0.25.0", @@ -2503,10 +1654,15 @@ } } }, + "node_modules/vite/node_modules/fsevents": { + "dev": true, + "optional": true + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, "license": "ISC" } } diff --git a/website/package.json b/website/package.json index 9394db0..560d53d 100644 --- a/website/package.json +++ b/website/package.json @@ -1,5 +1,5 @@ { - "name": "react-example", + "name": "agentic-webview-website", "private": true, "version": "0.0.0", "type": "module", @@ -11,24 +11,16 @@ "lint": "tsc --noEmit" }, "dependencies": { - "@tailwindcss/vite": "^4.1.14", - "@vitejs/plugin-react": "^5.0.4", - "dotenv": "^17.2.3", - "express": "^4.21.2", "lucide-react": "^0.546.0", - "motion": "^12.23.24", "react": "^19.0.1", "react-dom": "^19.0.1", - "react-router-dom": "^7.16.0", - "vite": "^6.2.3" + "react-router-dom": "^7.16.0" }, "devDependencies": { - "@types/express": "^4.17.21", + "@tailwindcss/vite": "^4.1.14", + "@vitejs/plugin-react": "^5.0.4", "@types/node": "^22.14.0", - "autoprefixer": "^10.4.21", - "esbuild": "^0.25.0", "tailwindcss": "^4.1.14", - "tsx": "^4.21.0", "typescript": "~5.8.2", "vite": "^6.2.3" } diff --git a/website/src/App.tsx b/website/src/App.tsx index eb5c74d..c60862a 100644 --- a/website/src/App.tsx +++ b/website/src/App.tsx @@ -1,10 +1,7 @@ import React from 'react'; import { BrowserRouter, Routes, Route } from 'react-router-dom'; import Home from './pages/Home'; -import DocumentationLayout from './pages/DocumentationLayout'; -import IntegrationGuide from './pages/docs/IntegrationGuide'; -import AgentIntegration from './pages/docs/AgentIntegration'; -import BestPractices from './pages/docs/BestPractices'; +import CanonicalDocs from './pages/CanonicalDocs'; import { ThemeProvider } from './components/ThemeProvider'; export default function App() { @@ -13,11 +10,7 @@ export default function App() { } /> - }> - } /> - } /> - } /> - + } /> diff --git a/website/src/components/DocPage.tsx b/website/src/components/DocPage.tsx deleted file mode 100644 index 7f0a359..0000000 --- a/website/src/components/DocPage.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import React from 'react'; -import { useLocation } from 'react-router-dom'; -import { SEO } from './SEO'; - -interface DocPageProps { - title: string; - description: React.ReactNode; - seoDescription?: string; - children: React.ReactNode; -} - -export function DocPage({ title, description, seoDescription, children }: DocPageProps) { - const location = useLocation(); - - const cleanDescription = seoDescription || - (typeof description === 'string' ? description : 'Agentic WebView SDK documentation and integration guide.'); - - return ( -
- -

{title}

-

- {description} -

- -
- {children} -
-
- ); -} diff --git a/website/src/components/InlineCode.tsx b/website/src/components/InlineCode.tsx deleted file mode 100644 index 080e485..0000000 --- a/website/src/components/InlineCode.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import React from 'react'; - -interface InlineCodeProps extends React.HTMLAttributes { - children: React.ReactNode; - className?: string; -} - -export function InlineCode({ children, className, ...props }: InlineCodeProps) { - // Always use text-[12px] as requested. Combine with the default styles. - const baseClasses = "bg-vp-input px-1.5 py-0.5 rounded border border-vp-border text-[12px]"; - - return ( - - {children} - - ); -} diff --git a/website/src/hooks/useLatestRelease.ts b/website/src/hooks/useLatestRelease.ts deleted file mode 100644 index ad23589..0000000 --- a/website/src/hooks/useLatestRelease.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { useState, useEffect } from 'react'; - -export function useLatestRelease() { - const [version, setVersion] = useState('0.2.1'); // default fallback - - useEffect(() => { - fetch('https://api.github.com/repos/shantoislamdev/agentic-webview/releases/latest') - .then((res) => { - if (!res.ok) { - console.warn('Could not fetch latest release (repository might be private). Using static fallback 0.2.1.'); - return null; - } - return res.json(); - }) - .then((data) => { - if (data && data.tag_name) { - // Sometimes tags start with 'v', we can keep it or remove it depending on standard. - // In Home.tsx, it's "v0.2.1" and "0.2.1", let's provide both. - const tag = data.tag_name; // e.g. "v0.2.1" or "0.2.1" - const cleanVersion = tag.startsWith('v') ? tag.slice(1) : tag; - setVersion(cleanVersion); - } - }) - .catch((err) => { - // Silently catch network errors to keep it clean in console - console.warn('Network error when fetching release, falling back to 0.2.1'); - }); - }, []); - - return { - version, - fetchVersion: () => version, - tagVersion: `v${version.replace(/^v/, '')}` - }; -} diff --git a/website/src/pages/CanonicalDocs.tsx b/website/src/pages/CanonicalDocs.tsx new file mode 100644 index 0000000..6c32e05 --- /dev/null +++ b/website/src/pages/CanonicalDocs.tsx @@ -0,0 +1,62 @@ +import React from 'react'; +import { Link } from 'react-router-dom'; +import { ArrowLeft, ArrowUpRight } from 'lucide-react'; +import { Footer } from '../components/Footer'; +import { Logo } from '../components/Logo'; +import { SEO } from '../components/SEO'; +import { ThemeToggle } from '../components/ThemeToggle'; + +const repository = 'https://github.com/shantoislamdev/agentic-webview/blob/main/docs'; +const documents = [ + ['Getting started', 'getting-started.md'], + ['Architecture', 'architecture.md'], + ['Android Views', 'views.md'], + ['Compose', 'compose.md'], + ['Agent tools', 'agent-tools.md'], + ['Koog adapter', 'koog.md'], + ['Observations', 'observations.md'], + ['Commands', 'commands.md'], + ['Lifecycle', 'lifecycle.md'], + ['Security', 'security.md'], + ['Frames and Shadow DOM', 'frames-shadow-dom.md'], + ['Privacy and prompt injection', 'privacy-prompt-injection.md'], + ['Diagnostics', 'diagnostics.md'], + ['Testing', 'testing.md'], + ['Release process', 'release.md'], +]; + +export default function CanonicalDocs() { + return ( +
+ +
+ + + Agentic WebView + + +
+
+ + Home + +

Documentation

+

+ The repository Markdown files are canonical so code examples cannot silently diverge between the website and the SDK. +

+
+ {documents.map(([label, file]) => ( + + {label} + + ))} +
+
+
+
+ ); +} diff --git a/website/src/pages/DocumentationLayout.tsx b/website/src/pages/DocumentationLayout.tsx deleted file mode 100644 index 2683e71..0000000 --- a/website/src/pages/DocumentationLayout.tsx +++ /dev/null @@ -1,148 +0,0 @@ -import React, { useState, useEffect } from 'react'; -import { NavLink, Outlet, Navigate, useLocation } from 'react-router-dom'; -import { ChevronRight, ArrowLeft, PanelLeftClose, PanelLeftOpen } from 'lucide-react'; -import { Link } from 'react-router-dom'; -import { ThemeToggle } from '../components/ThemeToggle'; -import { Footer } from '../components/Footer'; -import { Logo } from '../components/Logo'; - -const DOCS_NAV = [ - { path: '/documentation/integration-guide', label: 'Integration Guide' }, - { path: '/documentation/agent-integration', label: 'Agent Integration Guide' }, - { path: '/documentation/best-practices', label: 'Best Practices' }, -]; - -export default function DocumentationLayout() { - const location = useLocation(); - const [isSidebarOpen, setIsSidebarOpen] = useState(true); - - useEffect(() => { - const handleResize = () => { - if (window.innerWidth >= 768) { - setIsSidebarOpen(true); - } else { - setIsSidebarOpen(false); - } - }; - - // Set initial state based on window size - handleResize(); - - window.addEventListener('resize', handleResize); - return () => window.removeEventListener('resize', handleResize); - }, []); - - // Close menu when route changes on mobile - useEffect(() => { - if (window.innerWidth < 768) { - setIsSidebarOpen(false); - } - }, [location.pathname]); - - if (location.pathname === '/documentation' || location.pathname === '/documentation/') { - return ; - } - - const currentIndex = DOCS_NAV.findIndex(item => item.path === location.pathname); - const prevPage = currentIndex > 0 ? DOCS_NAV[currentIndex - 1] : null; - const nextPage = currentIndex < DOCS_NAV.length - 1 ? DOCS_NAV[currentIndex + 1] : null; - - const scrollToTop = () => { - document.getElementById('main-scroll-area')?.scrollTo({ top: 0, behavior: 'smooth' }); - }; - - return ( -
-
-
- - - - Agentic WebView - -
- -
- -
- {/* Sidebar Navigation */} -
- -
- - {/* Main Content Area */} -
- - - {/* Prev / Next Pagination */} -
-
- {prevPage && ( - - Previous - {prevPage.label} - - )} -
-
- {nextPage && ( - - Next - {nextPage.label} - - )} -
-
- -
-
-
-
- ); -} diff --git a/website/src/pages/Home.tsx b/website/src/pages/Home.tsx index 9520a79..352b10e 100644 --- a/website/src/pages/Home.tsx +++ b/website/src/pages/Home.tsx @@ -1,143 +1,90 @@ import React from 'react'; -import { CopyButton } from '../components/CopyButton'; import { Link } from 'react-router-dom'; -import { ThemeToggle } from '../components/ThemeToggle'; -import { Logo } from '../components/Logo'; +import { ArrowRight, ArrowUpRight } from 'lucide-react'; import { CodeSnippet } from '../components/CodeSnippet'; -import { InlineCode } from '../components/InlineCode'; import { Footer } from '../components/Footer'; -import { ArrowRight, ArrowUpRight } from 'lucide-react'; -import { useLatestRelease } from '../hooks/useLatestRelease'; +import { Logo } from '../components/Logo'; import { SEO } from '../components/SEO'; +import { ThemeToggle } from '../components/ThemeToggle'; -const integrationCode = `// initialize the controller -val controller = remember { AgenticWebController() } - -// attach to compose -AgenticWebViewComposable( - controller = controller, - modifier = Modifier.fillMaxSize(), - config = AgenticWebViewConfig(enableDebugLogging = true) +const quickStart = `val host = rememberAgenticBrowserHost( + AgenticBrowserConfiguration() ) -// listen to standard browser events -controller.state.collect { state -> - state?.let { - println("Tracking location: " + it.url) - } +LaunchedEffect(host) { + host.session.navigate( + NavigationRequest("https://example.com") + ) } -// perform agentic actions -val result = controller.executeAction( - AgentAction.Navigate("https://google.com") +AgenticBrowserView( + host = host, + modifier = Modifier.fillMaxSize() )`; -const inputCode = `[15]