From 91ea3f5984357af8680b1ad15072fe921b803fc3 Mon Sep 17 00:00:00 2001 From: rangarajan19 Date: Mon, 12 Oct 2026 21:57:16 +0530 Subject: [PATCH 1/3] Import REST tools from a HAR file (browser network capture) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets an operator capture a web app's traffic in DevTools ("Save all as HAR") and import it as REST tools when the API has no OpenAPI spec, Postman collection or docs. Requests are matched to the connector's base URL, static assets and WebSocket entries are skipped, and same-template requests (e.g. /orders/123 and /orders/456) collapse into one tool with an {id} parameter. A HAR carries the browser's live session: cookies, bearer tokens and API keys. The browser strips response bodies, cookies and credential-looking headers before the file ever leaves it (with a size check so an oversized capture is rejected in the panel instead of failing the upload with a 413), and the backend parser drops Cookie/Authorization/credential-named headers, query parameters and body fields on top of that — the connector's own configured auth is what actually authenticates the tool's calls, not anything recorded in the capture. Closes #1005 --- docs/api-reference.md | 8 + docs/connectors/rest.md | 25 ++ .../connectors/connectors.controller.spec.ts | 1 + .../src/connectors/connectors.controller.ts | 31 +- .../connectors/connectors.import-all.spec.ts | 1 + .../src/connectors/connectors.module.ts | 3 +- .../connectors.service.soap-test.spec.ts | 1 + .../src/connectors/import-spec-errors.spec.ts | 1 + .../parsers/__fixtures__/har/sample.har.json | 138 +++++++ .../src/connectors/parsers/har.parser.spec.ts | 78 ++++ .../src/connectors/parsers/har.parser.ts | 338 ++++++++++++++++++ .../src/connectors/secret-guard.flows.spec.ts | 1 + .../frontend/src/app/connectors/[id]/page.tsx | 140 +++++++- 13 files changed, 745 insertions(+), 21 deletions(-) create mode 100644 packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json create mode 100644 packages/backend/src/connectors/parsers/har.parser.spec.ts create mode 100644 packages/backend/src/connectors/parsers/har.parser.ts diff --git a/docs/api-reference.md b/docs/api-reference.md index 1c3a3feb8..1d9c9f861 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -146,6 +146,14 @@ curl -s http://localhost:4000/api/connectors/$ID/import \ -H "Authorization: Bearer $TOKEN" \ -H 'Content-Type: application/json' \ -d '{"source": "json", "content": "[{\"name\":\"tool\",\"description\":\"desc\",...}]"}' + +# From a HAR file (browser network capture) — the UI slims the file (strips +# response bodies, cookies and credential headers) before upload; do the same +# before sending one by hand. +curl -s http://localhost:4000/api/connectors/$ID/import \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"source": "har", "content": "{\"log\":{\"version\":\"1.2\",\"entries\":[...]}}"}' ``` ### Set Environment Variables diff --git a/docs/connectors/rest.md b/docs/connectors/rest.md index 6e94b00ef..6cd43d72d 100644 --- a/docs/connectors/rest.md +++ b/docs/connectors/rest.md @@ -125,6 +125,31 @@ curl -X POST https://api.example.com/users \ This generates a tool with `token`, `name`, and `email` as parameters. +### From a HAR file + +When an API has no OpenAPI spec, no Postman collection and no docs — just a web app +that calls it — capture the traffic instead: in your browser's DevTools, open the +Network tab, use the app for the flow you want, then right-click → "Save all as +HAR". Import that file from the connector's **Import Tools** panel (source "HAR +file (browser capture)"); the UI only accepts a file here, not pasted text. + +Only requests to the connector's own base URL are kept; everything else (other +origins, stylesheets, scripts, images, fonts, WebSocket upgrades) is skipped. +Repeated requests to the same path with a different id (`/orders/123`, +`/orders/456`) collapse into one tool with an `{id}` path parameter. Query +parameters and JSON or form bodies become tool parameters. + +A HAR carries your browser's session: cookies, bearer tokens and API keys. None +of that is ever uploaded. The browser strips response bodies, cookies and +credential-looking headers before the file leaves it, and the backend parser +drops `Cookie`, `Authorization` and any header, query parameter or body field +named like a credential (`token`, `secret`, `password`, `api_key`, `session`, +and similar) on top of that — the connector's own configured auth, not +anything recorded in the capture, is what actually authenticates the tool's +calls. A credential-named body field (e.g. a login form's `password`) still +becomes a parameter, just with no captured default, since the call needs that +value supplied at call time. + ### Custom JSON Definition Define tools directly as JSON: diff --git a/packages/backend/src/connectors/connectors.controller.spec.ts b/packages/backend/src/connectors/connectors.controller.spec.ts index 68501c4f8..542d25ebd 100644 --- a/packages/backend/src/connectors/connectors.controller.spec.ts +++ b/packages/backend/src/connectors/connectors.controller.spec.ts @@ -51,6 +51,7 @@ function buildController(overrides: { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine (overrides.mcpOAuthService ?? {}) as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/connectors.controller.ts b/packages/backend/src/connectors/connectors.controller.ts index 40b8c89fe..d9c0b558b 100644 --- a/packages/backend/src/connectors/connectors.controller.ts +++ b/packages/backend/src/connectors/connectors.controller.ts @@ -38,6 +38,7 @@ import { WsdlParser } from './parsers/wsdl.parser'; import { GraphqlParser } from './parsers/graphql.parser'; import { PostmanParser } from './parsers/postman.parser'; import { CurlParser } from './parsers/curl.parser'; +import { HarParser } from './parsers/har.parser'; import { describeSpecError, readRemoteSpec } from './parsers/spec-read-error'; import { McpClientEngine } from './engines/mcp-client.engine'; import { McpOAuthService } from './mcp-oauth.service'; @@ -352,12 +353,12 @@ class UpdateOAuth1ConfigDto { class ImportToolsDto { @ApiProperty({ - enum: ['openapi', 'wsdl', 'graphql', 'postman', 'curl', 'json', 'mcp', 'odata'], + enum: ['openapi', 'wsdl', 'graphql', 'postman', 'curl', 'har', 'json', 'mcp', 'odata'], description: 'Which parser to run on the supplied content/url.', example: 'openapi', }) @IsString() - source: 'openapi' | 'wsdl' | 'graphql' | 'postman' | 'curl' | 'json' | 'mcp' | 'odata'; + source: 'openapi' | 'wsdl' | 'graphql' | 'postman' | 'curl' | 'har' | 'json' | 'mcp' | 'odata'; @ApiPropertyOptional({ description: 'Inline spec content (JSON or YAML). Mutually exclusive with `url`.', @@ -580,6 +581,7 @@ export class ConnectorsController { private readonly graphqlParser: GraphqlParser, private readonly postmanParser: PostmanParser, private readonly curlParser: CurlParser, + private readonly harParser: HarParser, private readonly mcpClientEngine: McpClientEngine, private readonly mcpOAuthService: McpOAuthService, private readonly catalogResync: CatalogResyncService, @@ -1723,7 +1725,7 @@ export class ConnectorsController { @Post(':id/import') @ApiOperation({ - summary: 'Import tools from any source: OpenAPI, Postman, cURL, WSDL, GraphQL', + summary: 'Import tools from any source: OpenAPI, Postman, cURL, HAR, WSDL, GraphQL', }) async importTools( @Req() req: any, @@ -1739,6 +1741,7 @@ export class ConnectorsController { let odataScope: string | undefined; let detectedHealthcheckPath: string | undefined; + let harSkipped: { otherOrigin: number; staticAsset: number; duplicate: number } | undefined; try { switch (dto.source) { @@ -1786,6 +1789,16 @@ export class ConnectorsController { parsedTools = this.curlParser.parse(dto.content); break; } + case 'har': { + if (!dto.content) { + return { error: 'Provide the HAR file content in the content field' }; + } + const { baseUrl } = this.resolvedAddress(connector); + const result = this.harParser.parse(dto.content, { baseUrl }); + parsedTools = result.tools; + harSkipped = result.skipped; + break; + } case 'json': { if (!dto.content) { return { error: 'Provide the JSON tool definitions in content field' }; @@ -1888,8 +1901,8 @@ export class ConnectorsController { // Retiring everything else is what a full spec import (OpenAPI, WSDL…) // means; for a handful of hand-written tools it disabled every other tool // of the connector, catalog tools included. - const additive = dto.source === 'json' || dto.source === 'curl'; - return this.createToolsFromParsed( + const additive = dto.source === 'json' || dto.source === 'curl' || dto.source === 'har'; + const result = await this.createToolsFromParsed( connector, parsedTools, additive @@ -1901,6 +1914,14 @@ export class ConnectorsController { } : undefined, ); + if (harSkipped) { + return { + ...result, + message: `${result.message} (skipped ${harSkipped.otherOrigin} other-origin, ${harSkipped.staticAsset} static/socket, ${harSkipped.duplicate} duplicate entries)`, + harSkipped, + }; + } + return result; } @Put(':id/env-vars') diff --git a/packages/backend/src/connectors/connectors.import-all.spec.ts b/packages/backend/src/connectors/connectors.import-all.spec.ts index 2d4396256..40e987f39 100644 --- a/packages/backend/src/connectors/connectors.import-all.spec.ts +++ b/packages/backend/src/connectors/connectors.import-all.spec.ts @@ -89,6 +89,7 @@ function buildController(over: { prisma?: any; licenseGuard?: any } = {}) { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine {} as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/connectors.module.ts b/packages/backend/src/connectors/connectors.module.ts index 5b75df542..79e4e9642 100644 --- a/packages/backend/src/connectors/connectors.module.ts +++ b/packages/backend/src/connectors/connectors.module.ts @@ -16,6 +16,7 @@ import { WsdlParser } from './parsers/wsdl.parser'; import { GraphqlParser } from './parsers/graphql.parser'; import { PostmanParser } from './parsers/postman.parser'; import { CurlParser } from './parsers/curl.parser'; +import { HarParser } from './parsers/har.parser'; import { McpOAuthService } from './mcp-oauth.service'; import { McpOAuthCallbackController } from './mcp-oauth-callback.controller'; import { CatalogResyncService } from './catalog-resync.service'; @@ -32,7 +33,7 @@ const ENGINES = [ ODataEngine, ]; -const PARSERS = [OpenApiParser, WsdlParser, GraphqlParser, PostmanParser, CurlParser]; +const PARSERS = [OpenApiParser, WsdlParser, GraphqlParser, PostmanParser, CurlParser, HarParser]; // OAuth2TokenService comes from McpServerModule: one instance for the whole // app. A second one here had its own token cache and refresh mutex, so a diff --git a/packages/backend/src/connectors/connectors.service.soap-test.spec.ts b/packages/backend/src/connectors/connectors.service.soap-test.spec.ts index 62b6cc3b0..5f2dbec27 100644 --- a/packages/backend/src/connectors/connectors.service.soap-test.spec.ts +++ b/packages/backend/src/connectors/connectors.service.soap-test.spec.ts @@ -303,6 +303,7 @@ describe('ConnectorsService.testConnection for SOAP connectors', () => { {} as any, {} as any, {} as any, + {} as any, // harParser {} as any, {} as any, {} as any, diff --git a/packages/backend/src/connectors/import-spec-errors.spec.ts b/packages/backend/src/connectors/import-spec-errors.spec.ts index 4704c0a8f..fba82c824 100644 --- a/packages/backend/src/connectors/import-spec-errors.spec.ts +++ b/packages/backend/src/connectors/import-spec-errors.spec.ts @@ -33,6 +33,7 @@ describe('import-spec: an unreadable specification is a 422, not a 500', () => { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine {} as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json b/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json new file mode 100644 index 000000000..5077a0bfa --- /dev/null +++ b/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json @@ -0,0 +1,138 @@ +{ + "log": { + "version": "1.2", + "creator": { "name": "Fixture", "version": "1.0" }, + "entries": [ + { + "startedDateTime": "2026-10-12T10:00:00.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/orders/123?status=open&access_token=abc123", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Cookie", "value": "session=deadbeef" }, + { "name": "Authorization", "value": "Bearer eyJsupersecret" }, + { "name": "X-Api-Key", "value": "sk_live_abcdef123456" }, + { "name": "User-Agent", "value": "Mozilla/5.0" }, + { "name": "Accept", "value": "application/json" }, + { "name": "X-Org-Id", "value": "org_42" } + ], + "queryString": [ + { "name": "status", "value": "open" }, + { "name": "access_token", "value": "abc123" } + ], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": null + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [{ "name": "Content-Type", "value": "application/json" }], + "content": { "mimeType": "application/json", "size": 42, "text": "{\"id\":123}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:01.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/orders/456", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Cookie", "value": "session=deadbeef" }, + { "name": "Authorization", "value": "Bearer eyJsupersecret" } + ], + "queryString": [], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": null + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [], + "content": { "mimeType": "application/json", "size": 42, "text": "{\"id\":456}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:02.000Z", + "request": { + "method": "POST", + "url": "https://api.example.com/v2/login", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Content-Type", "value": "application/json" }, + { "name": "Cookie", "value": "session=deadbeef" } + ], + "queryString": [], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": { + "mimeType": "application/json", + "text": "{\"username\":\"bob\",\"password\":\"hunter2\"}" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [], + "content": { "mimeType": "application/json", "size": 2, "text": "{}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:03.000Z", + "request": { + "method": "GET", + "url": "https://other.example.com/track?x=1", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [{ "name": "x", "value": "1" }], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:04.000Z", + "_resourceType": "script", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/static/app.js", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:05.000Z", + "request": { + "method": "OPTIONS", + "url": "https://api.example.com/v2/orders/123", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 204, "statusText": "No Content", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:06.000Z", + "_resourceType": "websocket", + "request": { + "method": "GET", + "url": "wss://api.example.com/v2/socket", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 101, "statusText": "Switching Protocols", "headers": [], "content": {}, "cookies": [] } + } + ] + } +} diff --git a/packages/backend/src/connectors/parsers/har.parser.spec.ts b/packages/backend/src/connectors/parsers/har.parser.spec.ts new file mode 100644 index 000000000..85364e686 --- /dev/null +++ b/packages/backend/src/connectors/parsers/har.parser.spec.ts @@ -0,0 +1,78 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { HarParser } from './har.parser'; + +const FIXTURE = readFileSync(join(__dirname, '__fixtures__/har/sample.har.json'), 'utf-8'); + +describe('HarParser', () => { + let parser: HarParser; + + beforeEach(() => { + parser = new HarParser(); + }); + + it('collapses same-template requests, skips other origins, static assets and duplicates', () => { + const { tools, skipped } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + + expect(tools).toHaveLength(2); + expect(skipped).toEqual({ otherOrigin: 1, staticAsset: 2, duplicate: 1 }); + }); + + it('maps a GET with a path id and query parameters', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const orders = tools.find((t) => t.endpointMapping.path === '/orders/{id}'); + expect(orders).toBeDefined(); + expect(orders!.endpointMapping.method).toBe('GET'); + + const params = orders!.parameters as any; + expect(params.properties.id).toBeDefined(); + expect(params.required).toContain('id'); + expect(params.properties.status.default).toBe('open'); + expect(orders!.endpointMapping.queryParams).toEqual({ status: '$status' }); + expect(orders!.endpointMapping.headers).toEqual({ 'X-Org-Id': 'org_42' }); + }); + + it('respects a base URL with a path prefix', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const orders = tools.find((t) => t.endpointMapping.path === '/orders/{id}'); + // The fixture's requests are under /v2; the stored path is relative to it. + expect(orders!.endpointMapping.path.startsWith('/v2')).toBe(false); + }); + + it('maps a JSON body, marking a credential-named field as a parameter with no default', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const login = tools.find((t) => t.endpointMapping.path === '/login'); + expect(login).toBeDefined(); + expect(login!.endpointMapping.method).toBe('POST'); + + const params = login!.parameters as any; + expect(params.properties.username.default).toBe('bob'); + expect(params.properties.password).toBeDefined(); + expect(params.properties.password.default).toBeUndefined(); + expect(login!.endpointMapping.bodyMapping).toEqual({ username: '$username', password: '$password' }); + }); + + it('never lets a captured credential value survive into the parsed tools', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const serialized = JSON.stringify(tools); + expect(serialized).not.toContain('deadbeef'); // Cookie + expect(serialized).not.toContain('eyJsupersecret'); // Authorization + expect(serialized).not.toContain('sk_live_abcdef123456'); // X-Api-Key + expect(serialized).not.toContain('abc123'); // ?access_token= + expect(serialized).not.toContain('hunter2'); // body password field + }); + + it('is deterministic: parsing the same HAR twice yields the same tool names and shapes', () => { + const first = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const second = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + expect(second.tools).toEqual(first.tools); + }); + + it('rejects a file that is not JSON', () => { + expect(() => parser.parse('not json', { baseUrl: 'https://api.example.com/v2' })).toThrow(); + }); + + it('rejects valid JSON that is not a HAR (no log.entries)', () => { + expect(() => parser.parse('{"foo":"bar"}', { baseUrl: 'https://api.example.com/v2' })).toThrow(); + }); +}); diff --git a/packages/backend/src/connectors/parsers/har.parser.ts b/packages/backend/src/connectors/parsers/har.parser.ts new file mode 100644 index 000000000..c31a593b3 --- /dev/null +++ b/packages/backend/src/connectors/parsers/har.parser.ts @@ -0,0 +1,338 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ParsedTool } from './openapi.parser'; +import { capToolName, pathForToolName, MAX_TOOL_NAME_LENGTH } from './tool-name.util'; + +/** + * HAR (HTTP Archive) parser. + * Turns a browser network capture into REST tools, the same way "Copy as + * cURL" + the cURL parser turns one request into one tool, but for a whole + * recorded session. + * + * A HAR is live captured traffic, not a hand-written spec: it carries the + * browser's own session cookies, bearer tokens and auth headers. Those never + * become tool parameters or header mappings — the connector's own auth + * settings supply credentials at call time. Response bodies are never read. + */ + +export interface HarParseResult { + tools: ParsedTool[]; + skipped: { + otherOrigin: number; + staticAsset: number; + duplicate: number; + }; +} + +interface HarNameValue { + name: string; + value: string; +} + +interface HarPostData { + mimeType?: string; + text?: string; + params?: HarNameValue[]; +} + +interface HarRequest { + method?: string; + url?: string; + headers?: HarNameValue[]; + queryString?: HarNameValue[]; + postData?: HarPostData; +} + +interface HarEntry { + request?: HarRequest; + _resourceType?: string; + _webSocketMessages?: unknown[]; +} + +/** Fields whose name alone is enough to treat the value as a credential. */ +const CREDENTIAL_NAME_RE = /(token|secret|password|passwd|api[-_]?key|session|csrf|xsrf|auth|signature|sig)/i; + +const EXPLICIT_CREDENTIAL_HEADERS = new Set(['cookie', 'authorization', 'proxy-authorization']); + +/** Headers that describe the browser/request, not the API, and the API's own content metadata (redundant with postData). */ +const NOISE_HEADERS = new Set(['user-agent', 'origin', 'referer', 'content-length', 'content-type']); + +function isPseudoHeader(name: string): boolean { + return name.startsWith(':'); +} + +function isNoiseHeader(name: string): boolean { + const n = name.toLowerCase(); + return NOISE_HEADERS.has(n) || n.startsWith('accept') || n.startsWith('sec-'); +} + +function isCredentialHeaderName(name: string): boolean { + const n = name.toLowerCase(); + return EXPLICIT_CREDENTIAL_HEADERS.has(n) || isPseudoHeader(n) || CREDENTIAL_NAME_RE.test(n); +} + +function isCredentialFieldName(name: string): boolean { + return CREDENTIAL_NAME_RE.test(name); +} + +const STATIC_EXTENSION_RE = /\.(?:js|mjs|css|png|jpe?g|gif|svg|webp|avif|ico|woff2?|ttf|otf|eot|map|mp4|mp3|webm)(?:[?#]|$)/i; +const STATIC_RESOURCE_TYPES = new Set(['script', 'stylesheet', 'image', 'font', 'media', 'favicon', 'websocket']); + +function isStaticAssetOrSocket(entry: HarEntry, relativePath: string): boolean { + const resourceType = entry._resourceType?.toLowerCase(); + if (resourceType && STATIC_RESOURCE_TYPES.has(resourceType)) return true; + if (STATIC_EXTENSION_RE.test(relativePath)) return true; + if (Array.isArray(entry._webSocketMessages)) return true; + if (/^wss?:/i.test(entry.request?.url ?? '')) return true; + return false; +} + +/** `wss:`/`ws:` is `https:`/`http:` for origin matching: a capture's socket upgrade shares its page's origin. */ +function normalizedOrigin(url: URL): string { + const scheme = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol; + return `${scheme}//${url.host}`; +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const NUMERIC_RE = /^\d+$/; +const LONG_HEX_RE = /^[0-9a-f]{16,}$/i; + +function isIdLikeSegment(segment: string): boolean { + return NUMERIC_RE.test(segment) || UUID_RE.test(segment) || LONG_HEX_RE.test(segment); +} + +function sanitizeIdent(raw: string): string { + const cleaned = raw + .replace(/[^a-zA-Z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, '') + .toLowerCase(); + return cleaned; +} + +/** + * Replaces id-looking path segments with a named placeholder, so + * `/orders/123` and `/orders/456` collapse to one template with one `{id}` + * parameter. A second id-like segment in the same path gets `{id_2}`, and so on. + */ +function templatePath(pathname: string): { template: string; paramNames: string[] } { + const segments = pathname.split('/'); + const paramNames: string[] = []; + const usedParamNames = new Set(); + + const out = segments.map((segment) => { + if (segment.length > 0 && isIdLikeSegment(segment)) { + let name = 'id'; + let n = 2; + while (usedParamNames.has(name)) { + name = `id_${n++}`; + } + usedParamNames.add(name); + paramNames.push(name); + return `{${name}}`; + } + return segment; + }); + + return { template: out.join('/') || '/', paramNames }; +} + +function inferType(value: unknown): string { + if (typeof value === 'number') return 'number'; + if (typeof value === 'boolean') return 'boolean'; + if (Array.isArray(value)) return 'array'; + return 'string'; +} + +function generateToolName(method: string, template: string): string { + const cleanPath = pathForToolName(template) + .replace(/\{[^{}]+\}/g, '') + .replace(/[^a-zA-Z0-9]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, ''); + return `${method.toLowerCase()}_${cleanPath || 'request'}`; +} + +function uniqueName(base: string, used: Map): string { + const capped = capToolName(base); + const count = used.get(capped) ?? 0; + used.set(capped, count + 1); + if (count === 0) return capped; + const suffix = `_${count + 1}`; + const keep = Math.max(0, MAX_TOOL_NAME_LENGTH - suffix.length); + return `${capped.slice(0, keep)}${suffix}`; +} + +function buildTool( + method: string, + template: string, + paramNames: string[], + entry: HarEntry, + usedNames: Map, +): ParsedTool { + const properties: Record = {}; + const required: string[] = []; + const queryParamsMapping: Record = {}; + const bodyMapping: Record = {}; + const headerMapping: Record = {}; + + for (const paramName of paramNames) { + properties[paramName] = { type: 'string', description: `Path parameter: ${paramName}` }; + required.push(paramName); + } + + for (const header of entry.request?.headers ?? []) { + if (!header?.name || header.value == null) continue; + if (isCredentialHeaderName(header.name) || isNoiseHeader(header.name)) continue; + headerMapping[header.name] = header.value; + } + + for (const query of entry.request?.queryString ?? []) { + if (!query?.name) continue; + if (isCredentialFieldName(query.name)) continue; + const paramName = sanitizeIdent(query.name) || 'param'; + properties[paramName] = { + type: 'string', + description: `Query parameter: ${query.name}`, + default: query.value, + }; + queryParamsMapping[query.name] = `$${paramName}`; + } + + const postData = entry.request?.postData; + if (postData?.mimeType?.includes('application/json') && postData.text) { + try { + const parsed = JSON.parse(postData.text); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + for (const [key, value] of Object.entries(parsed)) { + const paramName = sanitizeIdent(key) || 'field'; + if (isCredentialFieldName(key)) { + properties[paramName] = { + type: 'string', + description: `Body field: ${key} (credential — value not captured from the recording)`, + }; + } else { + properties[paramName] = { + type: inferType(value), + description: `Body field: ${key}`, + default: value, + }; + } + bodyMapping[key] = `$${paramName}`; + } + } + } catch { + // Not a JSON object body — skip mapping, keep the tool otherwise usable. + } + } else if (postData?.mimeType?.includes('application/x-www-form-urlencoded') && Array.isArray(postData.params)) { + for (const field of postData.params) { + if (!field?.name) continue; + const paramName = sanitizeIdent(field.name) || 'field'; + if (isCredentialFieldName(field.name)) { + properties[paramName] = { + type: 'string', + description: `Body field: ${field.name} (credential — value not captured from the recording)`, + }; + } else { + properties[paramName] = { + type: 'string', + description: `Body field: ${field.name}`, + default: field.value, + }; + } + bodyMapping[field.name] = `$${paramName}`; + } + } + + const name = uniqueName(generateToolName(method, template), usedNames); + const parameters: Record = { type: 'object', properties }; + if (required.length > 0) parameters.required = required; + + const endpointMapping: ParsedTool['endpointMapping'] = { method, path: template }; + if (Object.keys(queryParamsMapping).length > 0) endpointMapping.queryParams = queryParamsMapping; + if (Object.keys(bodyMapping).length > 0) endpointMapping.bodyMapping = bodyMapping; + if (Object.keys(headerMapping).length > 0) endpointMapping.headers = headerMapping; + + return { + name, + description: `${method} ${template} (from HAR)`, + parameters, + endpointMapping, + }; +} + +@Injectable() +export class HarParser { + private readonly logger = new Logger(HarParser.name); + + parse(har: string, opts: { baseUrl: string }): HarParseResult { + let doc: any; + try { + doc = JSON.parse(har); + } catch (e: any) { + throw new Error(`Invalid HAR file: not valid JSON (${e.message})`); + } + + const entries: HarEntry[] = doc?.log?.entries; + if (!Array.isArray(entries)) { + throw new Error('Invalid HAR file: expected HAR 1.2 format with a log.entries array'); + } + + let base: URL; + try { + base = new URL(opts.baseUrl); + } catch { + throw new Error(`Invalid connector base URL: ${opts.baseUrl}`); + } + const basePrefix = base.pathname.replace(/\/$/, ''); + + const skipped = { otherOrigin: 0, staticAsset: 0, duplicate: 0 }; + const order: string[] = []; + const groups = new Map(); + + for (const entry of entries) { + const req = entry?.request; + if (!req?.url || !req?.method) continue; + const method = req.method.toUpperCase(); + if (method === 'OPTIONS' || method === 'HEAD') continue; + + let url: URL; + try { + url = new URL(req.url); + } catch { + continue; + } + + if (normalizedOrigin(url) !== normalizedOrigin(base) || !url.pathname.startsWith(basePrefix)) { + skipped.otherOrigin++; + continue; + } + + const relativePath = (basePrefix ? url.pathname.slice(basePrefix.length) : url.pathname) || '/'; + + if (isStaticAssetOrSocket(entry, relativePath)) { + skipped.staticAsset++; + continue; + } + + const { template, paramNames } = templatePath(relativePath); + const key = `${method} ${template}`; + if (groups.has(key)) { + skipped.duplicate++; + continue; + } + groups.set(key, { method, template, paramNames, entry }); + order.push(key); + } + + const usedNames = new Map(); + const tools: ParsedTool[] = order.map((key) => { + const g = groups.get(key)!; + return buildTool(g.method, g.template, g.paramNames, g.entry, usedNames); + }); + + this.logger.log( + `Parsed ${tools.length} tools from HAR (${skipped.otherOrigin} other origin, ${skipped.staticAsset} static/socket, ${skipped.duplicate} duplicate)`, + ); + return { tools, skipped }; + } +} diff --git a/packages/backend/src/connectors/secret-guard.flows.spec.ts b/packages/backend/src/connectors/secret-guard.flows.spec.ts index fb632f5ae..4506c4037 100644 --- a/packages/backend/src/connectors/secret-guard.flows.spec.ts +++ b/packages/backend/src/connectors/secret-guard.flows.spec.ts @@ -71,6 +71,7 @@ function buildConnectors(stored: Record = {}) { {} as any, new PostmanParser(), new CurlParser(), + {} as any, // harParser {} as any, {} as any, {} as any, diff --git a/packages/frontend/src/app/connectors/[id]/page.tsx b/packages/frontend/src/app/connectors/[id]/page.tsx index 2af37a1af..dd982b83c 100644 --- a/packages/frontend/src/app/connectors/[id]/page.tsx +++ b/packages/frontend/src/app/connectors/[id]/page.tsx @@ -24,6 +24,7 @@ const IMPORT_SOURCES = [ { id: 'openapi', label: 'OpenAPI / Swagger', placeholder: 'Paste OpenAPI JSON/YAML or enter URL...' }, { id: 'postman', label: 'Postman Collection', placeholder: 'Paste Postman Collection JSON or enter URL...' }, { id: 'curl', label: 'cURL Command', placeholder: 'curl -X GET https://api.example.com/users -H "Authorization: Bearer {{token}}"' }, + { id: 'har', label: 'HAR file (browser capture)', placeholder: '' }, { id: 'graphql', label: 'GraphQL Introspection', placeholder: 'Enter GraphQL endpoint URL...' }, { id: 'wsdl', label: 'WSDL', placeholder: 'Enter WSDL URL...' }, { id: 'json', label: 'JSON Definition', placeholder: '[\n {\n "name": "get_users",\n "description": "Fetch users",\n "parameters": { "type": "object", "properties": { "limit": { "type": "number" } } },\n "endpointMapping": { "method": "GET", "path": "/users", "queryParams": { "limit": "$limit" } }\n }\n]' }, @@ -44,6 +45,64 @@ function formatBytes(n: number): string { return `${(n / (1024 * 1024)).toFixed(1)} MB`; } +/** Field names a HAR entry carries that hold a credential by themselves. */ +const HAR_CREDENTIAL_HEADER_RE = /(token|secret|password|passwd|api[-_]?key|session|csrf|xsrf|auth|signature|sig)/i; +const HAR_EXPLICIT_CREDENTIAL_HEADERS = new Set(['cookie', 'set-cookie', 'authorization', 'proxy-authorization']); +const HAR_MAX_UPLOAD_BYTES = 7 * 1024 * 1024; + +function isHarCredentialHeader(name: string): boolean { + const n = name.toLowerCase(); + return HAR_EXPLICIT_CREDENTIAL_HEADERS.has(n) || n.startsWith(':') || HAR_CREDENTIAL_HEADER_RE.test(n); +} + +function slimHarHeaders(headers: unknown): any[] { + if (!Array.isArray(headers)) return []; + return headers.filter((h: any) => h && typeof h.name === 'string' && !isHarCredentialHeader(h.name)); +} + +/** + * Strips response bodies, cookies and credential headers from a HAR before + * it ever leaves the browser — a raw capture carries full response bodies + * and session cookies that must never be uploaded at all, on top of the + * credential headers/query/body fields the backend parser also scrubs. + */ +function slimHarForUpload(rawText: string): { json: string; byteLength: number } { + const har = JSON.parse(rawText); + const entries = har?.log?.entries; + if (!Array.isArray(entries)) { + throw new Error('Not a valid HAR file (expected a log.entries array)'); + } + const slimmed = { + log: { + version: har.log.version, + entries: entries.map((entry: any) => ({ + _resourceType: entry?._resourceType, + request: entry?.request + ? { + method: entry.request.method, + url: entry.request.url, + httpVersion: entry.request.httpVersion, + headers: slimHarHeaders(entry.request.headers), + queryString: entry.request.queryString, + postData: entry.request.postData, + } + : undefined, + response: entry?.response + ? { + status: entry.response.status, + headers: slimHarHeaders(entry.response.headers), + content: entry.response.content + ? { mimeType: entry.response.content.mimeType, size: entry.response.content.size } + : undefined, + } + : undefined, + })), + }, + }; + const json = JSON.stringify(slimmed); + return { json, byteLength: new TextEncoder().encode(json).length }; +} + export default function ConnectorDetailPage() { const { token } = useAuth(); const params = useParams(); @@ -152,6 +211,11 @@ export default function ConnectorDetailPage() { const [importUrl, setImportUrl] = useState(''); const [importing, setImporting] = useState(false); + // HAR import: the file is slimmed (response bodies, cookies, credential + // headers stripped) in the browser before its JSON lands in importContent. + const [harFileName, setHarFileName] = useState(''); + const [harError, setHarError] = useState(''); + // MCP assign modal — shown when tools are added and connector is not yet assigned const [showMcpAssign, setShowMcpAssign] = useState(false); @@ -482,6 +546,27 @@ export default function ConnectorDetailPage() { } }; + const handleHarFileChange = async (file: File | null) => { + setHarError(''); + setImportContent(''); + setHarFileName(''); + if (!file) return; + setHarFileName(file.name); + try { + const text = await file.text(); + const { json, byteLength } = slimHarForUpload(text); + if (byteLength > HAR_MAX_UPLOAD_BYTES) { + setHarError( + `This capture is still ${formatBytes(byteLength)} after removing response bodies and cookies, which is over what the server accepts. Re-capture a smaller session (fewer requests) and try again.`, + ); + return; + } + setImportContent(json); + } catch (err: any) { + setHarError(`Could not read this file: ${err.message}`); + } + }; + const handleImportTools = async () => { if (!token) return; setImporting(true); @@ -494,7 +579,7 @@ export default function ConnectorDetailPage() { if (importContent.trim()) data.content = importContent; const sets = importEntitySets.split(/[\s,]+/).map((x) => x.trim()).filter(Boolean); if (sets.length) data.entitySets = sets; - } else if (importSource === 'curl' || importSource === 'json') { + } else if (importSource === 'curl' || importSource === 'json' || importSource === 'har') { data.content = importContent; } else if (importUrl) { data.url = importUrl; @@ -509,6 +594,8 @@ export default function ConnectorDetailPage() { setShowImport(false); setImportContent(''); setImportUrl(''); + setHarFileName(''); + setHarError(''); fetchConnector(); promptMcpAssignIfNeeded(); } @@ -1606,7 +1693,7 @@ export default function ConnectorDetailPage() { {IMPORT_SOURCES.filter((src) => src.id !== 'odata' || hasOData(connector)).map((src) => (