diff --git a/docs/api-reference.md b/docs/api-reference.md index a347c0c6e..1db9dbc92 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -146,6 +146,14 @@ curl -s http://localhost:4000/api/connectors/$ID/import \ -H "Authorization: Bearer $TOKEN" \ -H 'Content-Type: application/json' \ -d '{"source": "json", "content": "[{\"name\":\"tool\",\"description\":\"desc\",...}]"}' + +# From a HAR file (browser network capture) — the UI slims the file (strips +# response bodies, cookies and credential headers) before upload; do the same +# before sending one by hand. +curl -s http://localhost:4000/api/connectors/$ID/import \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"source": "har", "content": "{\"log\":{\"version\":\"1.2\",\"entries\":[...]}}"}' ``` ### Set Environment Variables diff --git a/docs/connectors/rest.md b/docs/connectors/rest.md index 37b073753..0813eae39 100644 --- a/docs/connectors/rest.md +++ b/docs/connectors/rest.md @@ -125,6 +125,31 @@ curl -X POST https://api.example.com/users \ This generates a tool with `token`, `name`, and `email` as parameters. +### From a HAR file + +When an API has no OpenAPI spec, no Postman collection and no docs — just a web app +that calls it — capture the traffic instead: in your browser's DevTools, open the +Network tab, use the app for the flow you want, then right-click → "Save all as +HAR". Import that file from the connector's **Import Tools** panel (source "HAR +file (browser capture)"); the UI only accepts a file here, not pasted text. + +Only requests to the connector's own base URL are kept; everything else (other +origins, stylesheets, scripts, images, fonts, WebSocket upgrades) is skipped. +Repeated requests to the same path with a different id (`/orders/123`, +`/orders/456`) collapse into one tool with an `{id}` path parameter. Query +parameters and JSON or form bodies become tool parameters. + +A HAR carries your browser's session: cookies, bearer tokens and API keys. None +of that is ever uploaded. The browser strips response bodies, cookies and +credential-looking headers before the file leaves it, and the backend parser +drops `Cookie`, `Authorization` and any header, query parameter or body field +named like a credential (`token`, `secret`, `password`, `api_key`, `session`, +and similar) on top of that — the connector's own configured auth, not +anything recorded in the capture, is what actually authenticates the tool's +calls. A credential-named body field (e.g. a login form's `password`) still +becomes a parameter, just with no captured default, since the call needs that +value supplied at call time. + ### Custom JSON Definition Define tools directly as JSON: diff --git a/packages/backend/src/connectors/connectors.controller.spec.ts b/packages/backend/src/connectors/connectors.controller.spec.ts index cd7b4d594..35b41dfe1 100644 --- a/packages/backend/src/connectors/connectors.controller.spec.ts +++ b/packages/backend/src/connectors/connectors.controller.spec.ts @@ -51,6 +51,7 @@ function buildController(overrides: { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine (overrides.mcpOAuthService ?? {}) as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/connectors.controller.ts b/packages/backend/src/connectors/connectors.controller.ts index 25b76a7a8..44c72b02a 100644 --- a/packages/backend/src/connectors/connectors.controller.ts +++ b/packages/backend/src/connectors/connectors.controller.ts @@ -38,6 +38,7 @@ import { WsdlParser } from './parsers/wsdl.parser'; import { GraphqlParser } from './parsers/graphql.parser'; import { PostmanParser } from './parsers/postman.parser'; import { CurlParser } from './parsers/curl.parser'; +import { HarParser } from './parsers/har.parser'; import { describeSpecError, readRemoteSpec } from './parsers/spec-read-error'; import { McpClientEngine } from './engines/mcp-client.engine'; import { McpOAuthService } from './mcp-oauth.service'; @@ -381,12 +382,12 @@ class UpdateOAuth1ConfigDto { class ImportToolsDto { @ApiProperty({ - enum: ['openapi', 'wsdl', 'graphql', 'postman', 'curl', 'json', 'mcp', 'odata'], + enum: ['openapi', 'wsdl', 'graphql', 'postman', 'curl', 'har', 'json', 'mcp', 'odata'], description: 'Which parser to run on the supplied content/url.', example: 'openapi', }) @IsString() - source: 'openapi' | 'wsdl' | 'graphql' | 'postman' | 'curl' | 'json' | 'mcp' | 'odata'; + source: 'openapi' | 'wsdl' | 'graphql' | 'postman' | 'curl' | 'har' | 'json' | 'mcp' | 'odata'; @ApiPropertyOptional({ description: 'Inline spec content (JSON or YAML). Mutually exclusive with `url`.', @@ -609,6 +610,7 @@ export class ConnectorsController { private readonly graphqlParser: GraphqlParser, private readonly postmanParser: PostmanParser, private readonly curlParser: CurlParser, + private readonly harParser: HarParser, private readonly mcpClientEngine: McpClientEngine, private readonly mcpOAuthService: McpOAuthService, private readonly catalogResync: CatalogResyncService, @@ -1765,7 +1767,7 @@ export class ConnectorsController { @Post(':id/import') @ApiOperation({ - summary: 'Import tools from any source: OpenAPI, Postman, cURL, WSDL, GraphQL', + summary: 'Import tools from any source: OpenAPI, Postman, cURL, HAR, WSDL, GraphQL', }) async importTools( @Req() req: any, @@ -1781,6 +1783,7 @@ export class ConnectorsController { let odataScope: string | undefined; let detectedHealthcheckPath: string | undefined; + let harSkipped: { otherOrigin: number; staticAsset: number; duplicate: number } | undefined; try { switch (dto.source) { @@ -1828,6 +1831,16 @@ export class ConnectorsController { parsedTools = this.curlParser.parse(dto.content); break; } + case 'har': { + if (!dto.content) { + return { error: 'Provide the HAR file content in the content field' }; + } + const { baseUrl } = this.resolvedAddress(connector); + const result = this.harParser.parse(dto.content, { baseUrl }); + parsedTools = result.tools; + harSkipped = result.skipped; + break; + } case 'json': { if (!dto.content) { return { error: 'Provide the JSON tool definitions in content field' }; @@ -1930,8 +1943,8 @@ export class ConnectorsController { // Retiring everything else is what a full spec import (OpenAPI, WSDL…) // means; for a handful of hand-written tools it disabled every other tool // of the connector, catalog tools included. - const additive = dto.source === 'json' || dto.source === 'curl'; - return this.createToolsFromParsed( + const additive = dto.source === 'json' || dto.source === 'curl' || dto.source === 'har'; + const result = await this.createToolsFromParsed( connector, parsedTools, additive @@ -1943,6 +1956,14 @@ export class ConnectorsController { } : undefined, ); + if (harSkipped) { + return { + ...result, + message: `${result.message} (skipped ${harSkipped.otherOrigin} other-origin, ${harSkipped.staticAsset} static/socket, ${harSkipped.duplicate} duplicate entries)`, + harSkipped, + }; + } + return result; } @Put(':id/env-vars') diff --git a/packages/backend/src/connectors/connectors.import-all.spec.ts b/packages/backend/src/connectors/connectors.import-all.spec.ts index 2d4396256..40e987f39 100644 --- a/packages/backend/src/connectors/connectors.import-all.spec.ts +++ b/packages/backend/src/connectors/connectors.import-all.spec.ts @@ -89,6 +89,7 @@ function buildController(over: { prisma?: any; licenseGuard?: any } = {}) { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine {} as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/connectors.module.ts b/packages/backend/src/connectors/connectors.module.ts index 5b75df542..79e4e9642 100644 --- a/packages/backend/src/connectors/connectors.module.ts +++ b/packages/backend/src/connectors/connectors.module.ts @@ -16,6 +16,7 @@ import { WsdlParser } from './parsers/wsdl.parser'; import { GraphqlParser } from './parsers/graphql.parser'; import { PostmanParser } from './parsers/postman.parser'; import { CurlParser } from './parsers/curl.parser'; +import { HarParser } from './parsers/har.parser'; import { McpOAuthService } from './mcp-oauth.service'; import { McpOAuthCallbackController } from './mcp-oauth-callback.controller'; import { CatalogResyncService } from './catalog-resync.service'; @@ -32,7 +33,7 @@ const ENGINES = [ ODataEngine, ]; -const PARSERS = [OpenApiParser, WsdlParser, GraphqlParser, PostmanParser, CurlParser]; +const PARSERS = [OpenApiParser, WsdlParser, GraphqlParser, PostmanParser, CurlParser, HarParser]; // OAuth2TokenService comes from McpServerModule: one instance for the whole // app. A second one here had its own token cache and refresh mutex, so a diff --git a/packages/backend/src/connectors/connectors.service.soap-test.spec.ts b/packages/backend/src/connectors/connectors.service.soap-test.spec.ts index 62b6cc3b0..5f2dbec27 100644 --- a/packages/backend/src/connectors/connectors.service.soap-test.spec.ts +++ b/packages/backend/src/connectors/connectors.service.soap-test.spec.ts @@ -303,6 +303,7 @@ describe('ConnectorsService.testConnection for SOAP connectors', () => { {} as any, {} as any, {} as any, + {} as any, // harParser {} as any, {} as any, {} as any, diff --git a/packages/backend/src/connectors/import-spec-errors.spec.ts b/packages/backend/src/connectors/import-spec-errors.spec.ts index 4704c0a8f..fba82c824 100644 --- a/packages/backend/src/connectors/import-spec-errors.spec.ts +++ b/packages/backend/src/connectors/import-spec-errors.spec.ts @@ -33,6 +33,7 @@ describe('import-spec: an unreadable specification is a 422, not a 500', () => { {} as any, // graphqlParser {} as any, // postmanParser {} as any, // curlParser + {} as any, // harParser {} as any, // mcpClientEngine {} as any, // mcpOAuthService {} as any, // catalogResync diff --git a/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json b/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json new file mode 100644 index 000000000..f7d199d68 --- /dev/null +++ b/packages/backend/src/connectors/parsers/__fixtures__/har/sample.har.json @@ -0,0 +1,172 @@ +{ + "log": { + "version": "1.2", + "creator": { "name": "Fixture", "version": "1.0" }, + "entries": [ + { + "startedDateTime": "2026-10-12T10:00:00.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/orders/123?status=open&access_token=abc123", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Cookie", "value": "session=deadbeef" }, + { "name": "Authorization", "value": "Bearer eyJsupersecret" }, + { "name": "X-Api-Key", "value": "sk_live_abcdef123456" }, + { "name": "User-Agent", "value": "Mozilla/5.0" }, + { "name": "Accept", "value": "application/json" }, + { "name": "X-Org-Id", "value": "org_42" } + ], + "queryString": [ + { "name": "status", "value": "open" }, + { "name": "access_token", "value": "abc123" } + ], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": null + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [{ "name": "Content-Type", "value": "application/json" }], + "content": { "mimeType": "application/json", "size": 42, "text": "{\"id\":123}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:01.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/orders/456", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Cookie", "value": "session=deadbeef" }, + { "name": "Authorization", "value": "Bearer eyJsupersecret" } + ], + "queryString": [], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": null + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [], + "content": { "mimeType": "application/json", "size": 42, "text": "{\"id\":456}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:02.000Z", + "request": { + "method": "POST", + "url": "https://api.example.com/v2/login", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Content-Type", "value": "application/json" }, + { "name": "Cookie", "value": "session=deadbeef" } + ], + "queryString": [], + "cookies": [{ "name": "session", "value": "deadbeef" }], + "postData": { + "mimeType": "application/json", + "text": "{\"username\":\"bob\",\"password\":\"hunter2\",\"account\":{\"email\":\"bob@example.com\",\"password\":\"hunter2nested\"}}" + } + }, + "response": { + "status": 200, + "statusText": "OK", + "headers": [], + "content": { "mimeType": "application/json", "size": 2, "text": "{}" }, + "cookies": [] + } + }, + { + "startedDateTime": "2026-10-12T10:00:03.000Z", + "request": { + "method": "GET", + "url": "https://other.example.com/track?x=1", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [{ "name": "x", "value": "1" }], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:04.000Z", + "_resourceType": "script", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/static/app.js", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:05.000Z", + "request": { + "method": "OPTIONS", + "url": "https://api.example.com/v2/orders/123", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 204, "statusText": "No Content", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:06.000Z", + "_resourceType": "websocket", + "request": { + "method": "GET", + "url": "wss://api.example.com/v2/socket", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 101, "statusText": "Switching Protocols", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:07.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2/search?q=widgets&key=AIzaExampleKeyValue1234567890xyz", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Accept", "value": "application/json" }, + { "name": "Ocp-Apim-Subscription-Key", "value": "abcdef0123456789abcdef0123456789" }, + { "name": "X-Custom", "value": "Bearer abcDEF1234567890xyzLongEnough" }, + { "name": "X-Firebase-AppCheck", "value": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abc123signaturepart" } + ], + "queryString": [ + { "name": "q", "value": "widgets" }, + { "name": "key", "value": "AIzaExampleKeyValue1234567890xyz" } + ], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + }, + { + "startedDateTime": "2026-10-12T10:00:08.000Z", + "request": { + "method": "GET", + "url": "https://api.example.com/v2beta/widgets", + "httpVersion": "HTTP/1.1", + "headers": [], + "queryString": [], + "cookies": [], + "postData": null + }, + "response": { "status": 200, "statusText": "OK", "headers": [], "content": {}, "cookies": [] } + } + ] + } +} diff --git a/packages/backend/src/connectors/parsers/har.parser.spec.ts b/packages/backend/src/connectors/parsers/har.parser.spec.ts new file mode 100644 index 000000000..effb62a0e --- /dev/null +++ b/packages/backend/src/connectors/parsers/har.parser.spec.ts @@ -0,0 +1,176 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { HarParser } from './har.parser'; + +const FIXTURE = readFileSync(join(__dirname, '__fixtures__/har/sample.har.json'), 'utf-8'); + +describe('HarParser', () => { + let parser: HarParser; + + beforeEach(() => { + parser = new HarParser(); + }); + + it('collapses same-template requests, skips other origins, static assets and duplicates', () => { + const { tools, skipped } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + + expect(tools).toHaveLength(3); + // otherOrigin covers both a different host and /v2beta/x, which must not + // match the /v2 prefix on a non-boundary substring match. + expect(skipped).toEqual({ otherOrigin: 2, staticAsset: 2, duplicate: 1 }); + }); + + it('does not treat /v2beta as under the /v2 base prefix', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + expect(tools.some((t) => t.endpointMapping.path.includes('beta'))).toBe(false); + }); + + it('maps a GET with a path id and query parameters', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const orders = tools.find((t) => t.endpointMapping.path === '/orders/{id}'); + expect(orders).toBeDefined(); + expect(orders!.endpointMapping.method).toBe('GET'); + + const params = orders!.parameters as any; + expect(params.properties.id).toBeDefined(); + expect(params.required).toContain('id'); + expect(params.properties.status.default).toBe('open'); + expect(orders!.endpointMapping.queryParams).toEqual({ status: '$status' }); + expect(orders!.endpointMapping.headers).toEqual({ Accept: 'application/json', 'X-Org-Id': 'org_42' }); + }); + + it('respects a base URL with a path prefix', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const orders = tools.find((t) => t.endpointMapping.path === '/orders/{id}'); + // The fixture's requests are under /v2; the stored path is relative to it. + expect(orders!.endpointMapping.path.startsWith('/v2')).toBe(false); + }); + + it('maps a JSON body, marking a credential-named field as a parameter with no default', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const login = tools.find((t) => t.endpointMapping.path === '/login'); + expect(login).toBeDefined(); + expect(login!.endpointMapping.method).toBe('POST'); + + const params = login!.parameters as any; + expect(params.properties.username.default).toBe('bob'); + expect(params.properties.password).toBeDefined(); + expect(params.properties.password.default).toBeUndefined(); + }); + + it('recurses into a nested body object, dropping a nested credential but keeping its safe siblings', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const login = tools.find((t) => t.endpointMapping.path === '/login'); + const params = login!.parameters as any; + + // The nested object itself is never kept as a single opaque default — + // it is always walked into, so `account` never reaches inferType. + expect(params.properties.account).toBeUndefined(); + expect(params.properties.account_email.type).toBe('string'); + expect(params.properties.account_email.default).toBe('bob@example.com'); + expect(params.properties.account_password).toBeDefined(); + expect(params.properties.account_password.default).toBeUndefined(); + + expect(login!.endpointMapping.bodyMapping).toEqual({ + username: '$username', + password: '$password', + account: { email: '$account_email', password: '$account_password' }, + }); + }); + + it('sends a form body as a form, not as JSON', () => { + const har = (postData: Record) => + JSON.stringify({ + log: { + entries: [ + { + request: { + method: 'POST', + url: 'https://api.example.com/v2/oauth/form', + headers: [{ name: 'Content-Type', value: 'application/x-www-form-urlencoded' }], + postData, + }, + }, + ], + }, + }); + + const fromParams = parser.parse( + har({ + mimeType: 'application/x-www-form-urlencoded; charset=UTF-8', + params: [ + { name: 'grant_type', value: 'client_credentials' }, + { name: 'client_secret', value: 'hunter2' }, + ], + }), + { baseUrl: 'https://api.example.com/v2' }, + ).tools[0]; + expect(fromParams.endpointMapping.bodyEncoding).toBe('form-urlencoded'); + expect(fromParams.endpointMapping.bodyMapping).toEqual({ + grant_type: '$grant_type', + client_secret: '$client_secret', + }); + expect((fromParams.parameters as any).properties.client_secret.default).toBeUndefined(); + expect(JSON.stringify(fromParams)).not.toContain('hunter2'); + + // Some exporters only fill in `text` for a form body. + const fromText = parser.parse( + har({ mimeType: 'application/x-www-form-urlencoded', text: 'q=widgets&page=2' }), + { baseUrl: 'https://api.example.com/v2' }, + ).tools[0]; + expect(fromText.endpointMapping.bodyEncoding).toBe('form-urlencoded'); + expect(fromText.endpointMapping.bodyMapping).toEqual({ q: '$q', page: '$page' }); + expect((fromText.parameters as any).properties.page.default).toBe('2'); + }); + + it('leaves the default (JSON) encoding on a JSON body', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const login = tools.find((t) => t.endpointMapping.path === '/login'); + expect(login!.endpointMapping.bodyEncoding).toBeUndefined(); + }); + + it('drops credentials the issue\'s naming regex alone would miss', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const search = tools.find((t) => t.endpointMapping.path === '/search'); + expect(search).toBeDefined(); + + // Ocp-Apim-Subscription-Key: a vendor header whose name has no "api-key" substring. + // X-Custom: Bearer …: an auth-scheme value in a header name that looks like nothing special. + // X-Firebase-AppCheck: a bare JWT in a header name with no credential word at all. + expect(search!.endpointMapping.headers).toEqual({ Accept: 'application/json' }); + + // ?key=…: a bare "key" query name, not "api_key". + expect(search!.endpointMapping.queryParams).toEqual({ q: '$q' }); + const params = search!.parameters as any; + expect(params.properties.key).toBeUndefined(); + expect(params.properties.q.default).toBe('widgets'); + }); + + it('never lets a captured credential value survive into the parsed tools', () => { + const { tools } = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const serialized = JSON.stringify(tools); + expect(serialized).not.toContain('deadbeef'); // Cookie + expect(serialized).not.toContain('eyJsupersecret'); // Authorization + expect(serialized).not.toContain('sk_live_abcdef123456'); // X-Api-Key + expect(serialized).not.toContain('abc123'); // ?access_token= + expect(serialized).not.toContain('hunter2'); // body password field (top-level and nested) + expect(serialized).not.toContain('abcdef0123456789abcdef0123456789'); // Ocp-Apim-Subscription-Key + expect(serialized).not.toContain('abcDEF1234567890xyzLongEnough'); // X-Custom: Bearer … + expect(serialized).not.toContain('eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0'); // X-Firebase-AppCheck JWT + expect(serialized).not.toContain('AIzaExampleKeyValue1234567890xyz'); // ?key= + }); + + it('is deterministic: parsing the same HAR twice yields the same tool names and shapes', () => { + const first = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + const second = parser.parse(FIXTURE, { baseUrl: 'https://api.example.com/v2' }); + expect(second.tools).toEqual(first.tools); + }); + + it('rejects a file that is not JSON', () => { + expect(() => parser.parse('not json', { baseUrl: 'https://api.example.com/v2' })).toThrow(); + }); + + it('rejects valid JSON that is not a HAR (no log.entries)', () => { + expect(() => parser.parse('{"foo":"bar"}', { baseUrl: 'https://api.example.com/v2' })).toThrow(); + }); +}); diff --git a/packages/backend/src/connectors/parsers/har.parser.ts b/packages/backend/src/connectors/parsers/har.parser.ts new file mode 100644 index 000000000..79c260260 --- /dev/null +++ b/packages/backend/src/connectors/parsers/har.parser.ts @@ -0,0 +1,453 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ParsedTool } from './openapi.parser'; +import { capToolName, pathForToolName, MAX_TOOL_NAME_LENGTH } from './tool-name.util'; +import { + findSecretLiterals, + findSecretInQueryParam, + isCredentialHeaderName as isSharedCredentialHeaderName, + isSecretQueryName, +} from '../../common/secret-literals.util'; + +/** + * HAR (HTTP Archive) parser. + * Turns a browser network capture into REST tools, the same way "Copy as + * cURL" + the cURL parser turns one request into one tool, but for a whole + * recorded session. + * + * A HAR is live captured traffic, not a hand-written spec: it carries the + * browser's own session cookies, bearer tokens and auth headers, in forms the + * issue's own naming regex does not anticipate (`Ocp-Apim-Subscription-Key`, + * `?key=AIza…`, a bare JWT in a vendor-specific header). None of that becomes + * a tool parameter, default or header mapping — the connector's own auth + * settings supply credentials at call time, and a captured value belongs to + * the user's browser session, never to the connector. Credentials are + * dropped outright here, not promoted into connector variables the way the + * generic import pipeline moves a hand-typed key: that pipeline still runs + * after this parser (defense in depth), but by then nothing should be left + * for it to find. Response bodies are never read. + */ + +export interface HarParseResult { + tools: ParsedTool[]; + skipped: { + otherOrigin: number; + staticAsset: number; + duplicate: number; + }; +} + +interface HarNameValue { + name: string; + value: string; +} + +interface HarPostData { + mimeType?: string; + text?: string; + params?: HarNameValue[]; +} + +interface HarRequest { + method?: string; + url?: string; + headers?: HarNameValue[]; + queryString?: HarNameValue[]; + postData?: HarPostData; +} + +interface HarEntry { + request?: HarRequest; + _resourceType?: string; + _webSocketMessages?: unknown[]; +} + +/** Fields whose name alone is enough to treat the value as a credential (query params, body fields). */ +const CREDENTIAL_NAME_RE = /(token|secret|password|passwd|api[-_]?key|session|csrf|xsrf|auth|signature|sig)/i; + +function isCredentialFieldName(name: string): boolean { + return CREDENTIAL_NAME_RE.test(name); +} + +/** A value shaped like a bearer/basic/etc. credential, whatever its header or field is named. */ +const AUTH_SCHEME_VALUE_RE = /^(Bearer|Basic|Token|Bot|ApiKey|Api-Key|Key|OAuth|SSWS|Digest)\s+\S+$/i; + +/** A string value that looks like a credential by its shape, not just its name: a JWT, a known vendor key prefix, a bot token, or an auth-scheme value. */ +function looksLikeCredentialValue(value: unknown): boolean { + if (typeof value !== 'string' || !value) return false; + if (AUTH_SCHEME_VALUE_RE.test(value.trim())) return true; + return findSecretLiterals(value, { query: false }).length > 0; +} + +/** + * Headers worth replaying as static tool headers: `Accept` and vendor-specific + * `X-…` headers, once those pass the credential checks below. Everything else + * — conditional/caching headers (`If-None-Match`, `Cache-Control`), tracing + * headers (`traceparent`, `baggage`), and ordinary browser noise — is left + * out. An allowlist, not a denylist: a header this parser has never seen is + * excluded by default, not included by default. + */ +const HEADER_ALLOW_RE = /^(accept|x-[a-z0-9-]+)$/i; + +function isPseudoHeader(name: string): boolean { + return name.startsWith(':'); +} + +function shouldKeepHeader(name: string, value: string): boolean { + if (isPseudoHeader(name)) return false; + if (!HEADER_ALLOW_RE.test(name)) return false; + if (CREDENTIAL_NAME_RE.test(name) || isSharedCredentialHeaderName(name)) return false; + if (looksLikeCredentialValue(value)) return false; + return true; +} + +const STATIC_EXTENSION_RE = /\.(?:js|mjs|css|png|jpe?g|gif|svg|webp|avif|ico|woff2?|ttf|otf|eot|map|mp4|mp3|webm)(?:[?#]|$)/i; +const STATIC_RESOURCE_TYPES = new Set(['script', 'stylesheet', 'image', 'font', 'media', 'favicon', 'websocket']); + +function isStaticAssetOrSocket(entry: HarEntry, relativePath: string): boolean { + const resourceType = entry._resourceType?.toLowerCase(); + if (resourceType && STATIC_RESOURCE_TYPES.has(resourceType)) return true; + if (STATIC_EXTENSION_RE.test(relativePath)) return true; + if (Array.isArray(entry._webSocketMessages)) return true; + if (/^wss?:/i.test(entry.request?.url ?? '')) return true; + return false; +} + +/** `wss:`/`ws:` is `https:`/`http:` for origin matching: a capture's socket upgrade shares its page's origin. */ +function normalizedOrigin(url: URL): string { + const scheme = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol; + return `${scheme}//${url.host}`; +} + +/** Whether `pathname` is `prefix` or sits under it on a `/` boundary — `/v1` must not match into `/v1beta/x`. */ +function pathHasPrefix(pathname: string, prefix: string): boolean { + if (!prefix) return true; + return pathname === prefix || pathname.startsWith(`${prefix}/`); +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const NUMERIC_RE = /^\d+$/; +const LONG_HEX_RE = /^[0-9a-f]{16,}$/i; + +function isIdLikeSegment(segment: string): boolean { + return NUMERIC_RE.test(segment) || UUID_RE.test(segment) || LONG_HEX_RE.test(segment); +} + +function sanitizeIdent(raw: string): string { + const cleaned = raw + .replace(/[^a-zA-Z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, '') + .toLowerCase(); + return cleaned; +} + +/** First unused name among `base`, `base_2`, `base_3`, … in `used`, reserving it. */ +function ensureUniqueParamName(base: string, used: Set): string { + let name = base || 'param'; + let n = 2; + while (used.has(name)) { + name = `${base || 'param'}_${n++}`; + } + used.add(name); + return name; +} + +/** + * Replaces id-looking path segments with a named placeholder, so + * `/orders/123` and `/orders/456` collapse to one template with one `{id}` + * parameter. A second id-like segment in the same path gets `{id_2}`, and so on. + */ +function templatePath(pathname: string): { template: string; paramNames: string[] } { + const segments = pathname.split('/'); + const paramNames: string[] = []; + const usedParamNames = new Set(); + + const out = segments.map((segment) => { + if (segment.length > 0 && isIdLikeSegment(segment)) { + let name = 'id'; + let n = 2; + while (usedParamNames.has(name)) { + name = `id_${n++}`; + } + usedParamNames.add(name); + paramNames.push(name); + return `{${name}}`; + } + return segment; + }); + + return { template: out.join('/') || '/', paramNames }; +} + +function inferType(value: unknown): string { + if (typeof value === 'number') return 'number'; + if (typeof value === 'boolean') return 'boolean'; + if (Array.isArray(value)) return 'array'; + return 'string'; +} + +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +/** Whether any string anywhere inside `value` (object, array or scalar) looks like a credential. */ +function containsCredentialValue(value: unknown, depth = 0): boolean { + if (depth > 10) return false; + if (typeof value === 'string') return looksLikeCredentialValue(value); + if (Array.isArray(value)) return value.some((v) => containsCredentialValue(v, depth + 1)); + if (isPlainObject(value)) { + return Object.entries(value).some( + ([k, v]) => isCredentialFieldName(k) || containsCredentialValue(v, depth + 1), + ); + } + return false; +} + +function generateToolName(method: string, template: string): string { + const cleanPath = pathForToolName(template) + .replace(/\{[^{}]+\}/g, '') + .replace(/[^a-zA-Z0-9]/g, '_') + .replace(/_+/g, '_') + .replace(/^_+|_+$/g, ''); + return `${method.toLowerCase()}_${cleanPath || 'request'}`; +} + +function uniqueName(base: string, used: Map): string { + const capped = capToolName(base); + const count = used.get(capped) ?? 0; + used.set(capped, count + 1); + if (count === 0) return capped; + const suffix = `_${count + 1}`; + const keep = Math.max(0, MAX_TOOL_NAME_LENGTH - suffix.length); + return `${capped.slice(0, keep)}${suffix}`; +} + +const MAX_BODY_WALK_DEPTH = 6; + +/** + * Recursively walks one JSON body value, writing scrubbed leaves into + * `properties`/`bodyTemplate`. A plain object is never itself a leaf — it is + * always walked into — so a nested secret (`user.password`) is caught and + * dropped individually instead of the whole `user` object surviving as one + * opaque default, and no object ever reaches `inferType` to be mistyped as a + * `string`. Arrays and scalars are leaves: an array is scrubbed as a whole + * (dropped if any element looks like a credential) rather than indexed. + */ +function walkJsonBody( + value: unknown, + keyChain: string[], + properties: Record, + usedParamNames: Set, + depth: number, +): unknown { + if (isPlainObject(value) && depth < MAX_BODY_WALK_DEPTH) { + const template: Record = {}; + for (const [key, child] of Object.entries(value)) { + template[key] = walkJsonBody(child, [...keyChain, key], properties, usedParamNames, depth + 1); + } + return template; + } + + const lastKey = keyChain[keyChain.length - 1] ?? 'field'; + const label = keyChain.join('.'); + const isCredential = isCredentialFieldName(lastKey) || containsCredentialValue(value); + const paramName = ensureUniqueParamName(sanitizeIdent(keyChain.join('_')) || 'field', usedParamNames); + + if (isCredential) { + properties[paramName] = { + type: 'string', + description: `Body field: ${label} (credential — value not captured from the recording)`, + }; + } else { + properties[paramName] = { + type: inferType(value), + description: `Body field: ${label}`, + default: value, + }; + } + return `$${paramName}`; +} + +function buildTool( + method: string, + template: string, + paramNames: string[], + entry: HarEntry, + usedNames: Map, +): ParsedTool { + const properties: Record = {}; + const required: string[] = []; + const queryParamsMapping: Record = {}; + const bodyMapping: Record = {}; + const headerMapping: Record = {}; + const usedParamNames = new Set(paramNames); + + for (const paramName of paramNames) { + properties[paramName] = { type: 'string', description: `Path parameter: ${paramName}` }; + required.push(paramName); + } + + for (const header of entry.request?.headers ?? []) { + if (!header?.name || header.value == null) continue; + if (!shouldKeepHeader(header.name, header.value)) continue; + headerMapping[header.name] = header.value; + } + + for (const query of entry.request?.queryString ?? []) { + if (!query?.name) continue; + const value = query.value ?? ''; + if ( + isCredentialFieldName(query.name) || + isSecretQueryName(query.name) || + findSecretInQueryParam(query.name, value).length > 0 || + looksLikeCredentialValue(value) + ) { + continue; + } + const paramName = ensureUniqueParamName(sanitizeIdent(query.name) || 'param', usedParamNames); + properties[paramName] = { + type: 'string', + description: `Query parameter: ${query.name}`, + default: value, + }; + queryParamsMapping[query.name] = `$${paramName}`; + } + + const postData = entry.request?.postData; + let bodyEncoding: string | undefined; + if (postData?.mimeType?.includes('application/json') && postData.text) { + try { + const parsed = JSON.parse(postData.text); + if (isPlainObject(parsed)) { + for (const [key, value] of Object.entries(parsed)) { + bodyMapping[key] = walkJsonBody(value, [key], properties, usedParamNames, 1); + } + } + } catch { + // Not a JSON object body — skip mapping, keep the tool otherwise usable. + } + } else if (postData?.mimeType?.includes('application/x-www-form-urlencoded')) { + // Browsers export a form body as `params`; fall back to parsing `text` + // for tools that only fill that in. + const fields: HarNameValue[] = Array.isArray(postData.params) + ? postData.params + : postData.text + ? [...new URLSearchParams(postData.text)].map(([name, value]) => ({ name, value })) + : []; + // Content-Type is not a replayed header, so the engine needs to be told + // to send these fields as a form, not as JSON. + if (fields.length > 0) bodyEncoding = 'form-urlencoded'; + for (const field of fields) { + if (!field?.name) continue; + const value = field.value ?? ''; + const paramName = ensureUniqueParamName(sanitizeIdent(field.name) || 'field', usedParamNames); + if (isCredentialFieldName(field.name) || looksLikeCredentialValue(value)) { + properties[paramName] = { + type: 'string', + description: `Body field: ${field.name} (credential — value not captured from the recording)`, + }; + } else { + properties[paramName] = { + type: 'string', + description: `Body field: ${field.name}`, + default: value, + }; + } + bodyMapping[field.name] = `$${paramName}`; + } + } + + const name = uniqueName(generateToolName(method, template), usedNames); + const parameters: Record = { type: 'object', properties }; + if (required.length > 0) parameters.required = required; + + const endpointMapping: ParsedTool['endpointMapping'] = { method, path: template }; + if (Object.keys(queryParamsMapping).length > 0) endpointMapping.queryParams = queryParamsMapping; + if (Object.keys(bodyMapping).length > 0) endpointMapping.bodyMapping = bodyMapping; + if (bodyEncoding && endpointMapping.bodyMapping) endpointMapping.bodyEncoding = bodyEncoding; + if (Object.keys(headerMapping).length > 0) endpointMapping.headers = headerMapping; + + return { + name, + description: `${method} ${template} (from HAR)`, + parameters, + endpointMapping, + }; +} + +@Injectable() +export class HarParser { + private readonly logger = new Logger(HarParser.name); + + parse(har: string, opts: { baseUrl: string }): HarParseResult { + let doc: any; + try { + doc = JSON.parse(har); + } catch (e: any) { + throw new Error(`Invalid HAR file: not valid JSON (${e.message})`); + } + + const entries: HarEntry[] = doc?.log?.entries; + if (!Array.isArray(entries)) { + throw new Error('Invalid HAR file: expected HAR 1.2 format with a log.entries array'); + } + + let base: URL; + try { + base = new URL(opts.baseUrl); + } catch { + throw new Error(`Invalid connector base URL: ${opts.baseUrl}`); + } + const basePrefix = base.pathname.replace(/\/$/, ''); + + const skipped = { otherOrigin: 0, staticAsset: 0, duplicate: 0 }; + const order: string[] = []; + const groups = new Map(); + + for (const entry of entries) { + const req = entry?.request; + if (!req?.url || !req?.method) continue; + const method = req.method.toUpperCase(); + if (method === 'OPTIONS' || method === 'HEAD') continue; + + let url: URL; + try { + url = new URL(req.url); + } catch { + continue; + } + + if (normalizedOrigin(url) !== normalizedOrigin(base) || !pathHasPrefix(url.pathname, basePrefix)) { + skipped.otherOrigin++; + continue; + } + + const relativePath = (basePrefix ? url.pathname.slice(basePrefix.length) : url.pathname) || '/'; + + if (isStaticAssetOrSocket(entry, relativePath)) { + skipped.staticAsset++; + continue; + } + + const { template, paramNames } = templatePath(relativePath); + const key = `${method} ${template}`; + if (groups.has(key)) { + skipped.duplicate++; + continue; + } + groups.set(key, { method, template, paramNames, entry }); + order.push(key); + } + + const usedNames = new Map(); + const tools: ParsedTool[] = order.map((key) => { + const g = groups.get(key)!; + return buildTool(g.method, g.template, g.paramNames, g.entry, usedNames); + }); + + this.logger.log( + `Parsed ${tools.length} tools from HAR (${skipped.otherOrigin} other origin, ${skipped.staticAsset} static/socket, ${skipped.duplicate} duplicate)`, + ); + return { tools, skipped }; + } +} diff --git a/packages/backend/src/connectors/parsers/openapi.parser.ts b/packages/backend/src/connectors/parsers/openapi.parser.ts index 34b649526..c23ea4cf8 100644 --- a/packages/backend/src/connectors/parsers/openapi.parser.ts +++ b/packages/backend/src/connectors/parsers/openapi.parser.ts @@ -25,6 +25,7 @@ export interface ParsedTool { path: string; queryParams?: Record; bodyMapping?: Record; + bodyEncoding?: string; headers?: Record; }; responseMapping?: { diff --git a/packages/backend/src/connectors/secret-guard.flows.spec.ts b/packages/backend/src/connectors/secret-guard.flows.spec.ts index fb632f5ae..43fa11a00 100644 --- a/packages/backend/src/connectors/secret-guard.flows.spec.ts +++ b/packages/backend/src/connectors/secret-guard.flows.spec.ts @@ -2,6 +2,7 @@ import { ConnectorsController } from './connectors.controller'; import { ToolsController } from './tools.controller'; import { CurlParser } from './parsers/curl.parser'; import { PostmanParser } from './parsers/postman.parser'; +import { HarParser } from './parsers/har.parser'; import { AdaptersService } from '../adapters/adapters.service'; import { ConnectorSetupService } from '../adapters/connector-setup.service'; import { getAdapter } from '../adapters/catalog'; @@ -30,7 +31,7 @@ function noSecret(value: unknown): void { expect(text).not.toContain(GOOGLE); } -function buildConnectors(stored: Record = {}) { +function buildConnectors(stored: Record = {}, overrides: { harParser?: any } = {}) { const connector = { id: 'c1', name: 'Telegram', @@ -71,6 +72,7 @@ function buildConnectors(stored: Record = {}) { {} as any, new PostmanParser(), new CurlParser(), + overrides.harParser ?? ({} as any), {} as any, {} as any, {} as any, @@ -235,6 +237,80 @@ describe('imports', () => { noSecret(created); }); + it('HAR: captured headers, query and nested body credentials are dropped, never promoted to a connector variable', async () => { + const { controller, prisma, connectorsService } = buildConnectors( + { baseUrl: 'https://api.example.com' }, + { harParser: new HarParser() }, + ); + const har = { + log: { + version: '1.2', + entries: [ + { + request: { + method: 'POST', + url: `https://api.example.com/profile?key=${GOOGLE}`, + headers: [ + { name: 'Cookie', value: 'session=abc' }, + { name: 'Authorization', value: `Bearer ${TG}` }, + { name: 'Ocp-Apim-Subscription-Key', value: 'abcdef0123456789abcdef0123456789' }, + { name: 'X-Custom', value: 'Bearer abcDEF1234567890xyzLongEnough' }, + ], + queryString: [{ name: 'key', value: GOOGLE }], + postData: { + mimeType: 'application/json', + text: JSON.stringify({ user: { name: 'bob', password: 'hunter2' } }), + }, + }, + }, + ], + }, + }; + // A HAR is the user's own browser session, not a hand-typed secret: a + // captured credential must be dropped outright, never turned into a + // connector-wide variable the way cURL/Postman/JSON imports do above. + const out: any = await controller.importTools(req(), 'c1', { source: 'har', content: JSON.stringify(har) } as any); + expect(connectorsService.addEnvVars).not.toHaveBeenCalled(); + expect(out.secretsMoved).toBeUndefined(); + + const created = prisma.mcpTool.create.mock.calls[0][0].data; + expect(created.endpointMapping.headers).toBeUndefined(); + expect(created.endpointMapping.queryParams).toBeUndefined(); + expect(created.parameters.properties.user_name.default).toBe('bob'); + expect(created.parameters.properties.user_password.default).toBeUndefined(); + noSecret(created); + const serialized = JSON.stringify(created); + expect(serialized).not.toContain(TG); + expect(serialized).not.toContain(GOOGLE); + expect(serialized).not.toContain('hunter2'); + }); + + it('HAR: a token in the request path moves into a variable, as with cURL', async () => { + const { controller, prisma, connectorsService } = buildConnectors( + { baseUrl: 'https://api.telegram.org' }, + { harParser: new HarParser() }, + ); + const har = { + log: { + entries: [ + { + request: { + method: 'POST', + url: `https://api.telegram.org/bot${TG}/sendMessage`, + headers: [], + postData: { mimeType: 'application/json', text: '{"chat_id":"1","text":"hi"}' }, + }, + }, + ], + }, + }; + await controller.importTools(req(), 'c1', { source: 'har', content: JSON.stringify(har) } as any); + const created = prisma.mcpTool.create.mock.calls[0][0].data; + expect(created.endpointMapping.path).toBe('/bot{{TELEGRAM_BOT_TOKEN}}/sendMessage'); + expect(connectorsService.addEnvVars).toHaveBeenCalledWith('c1', { TELEGRAM_BOT_TOKEN: TG }); + noSecret(created); + }); + it('a DATABASE connector keeps its SQL as written', async () => { const { controller, prisma, connectorsService } = buildConnectors({ type: 'DATABASE' }); const sql = `SELECT * FROM t WHERE k = '${TG}'`; diff --git a/packages/frontend/src/app/connectors/[id]/page.tsx b/packages/frontend/src/app/connectors/[id]/page.tsx index 0b0ba2883..a01a17d69 100644 --- a/packages/frontend/src/app/connectors/[id]/page.tsx +++ b/packages/frontend/src/app/connectors/[id]/page.tsx @@ -24,6 +24,7 @@ const IMPORT_SOURCES = [ { id: 'openapi', label: 'OpenAPI / Swagger', placeholder: 'Paste OpenAPI JSON/YAML or enter URL...' }, { id: 'postman', label: 'Postman Collection', placeholder: 'Paste Postman Collection JSON or enter URL...' }, { id: 'curl', label: 'cURL Command', placeholder: 'curl -X GET https://api.example.com/users -H "Authorization: Bearer {{token}}"' }, + { id: 'har', label: 'HAR file (browser capture)', placeholder: '' }, { id: 'graphql', label: 'GraphQL Introspection', placeholder: 'Enter GraphQL endpoint URL...' }, { id: 'wsdl', label: 'WSDL', placeholder: 'Enter WSDL URL...' }, { id: 'json', label: 'JSON Definition', placeholder: '[\n {\n "name": "get_users",\n "description": "Fetch users",\n "parameters": { "type": "object", "properties": { "limit": { "type": "number" } } },\n "endpointMapping": { "method": "GET", "path": "/users", "queryParams": { "limit": "$limit" } }\n }\n]' }, @@ -44,6 +45,115 @@ function formatBytes(n: number): string { return `${(n / (1024 * 1024)).toFixed(1)} MB`; } +/** Field names a HAR entry carries that hold a credential by themselves. */ +const HAR_CREDENTIAL_HEADER_RE = /(token|secret|password|passwd|api[-_]?key|session|csrf|xsrf|auth|signature|sig)/i; +const HAR_EXPLICIT_CREDENTIAL_HEADERS = new Set(['cookie', 'set-cookie', 'authorization', 'proxy-authorization']); +const HAR_MAX_UPLOAD_BYTES = 7 * 1024 * 1024; + +function isHarCredentialHeader(name: string): boolean { + const n = name.toLowerCase(); + return HAR_EXPLICIT_CREDENTIAL_HEADERS.has(n) || n.startsWith(':') || HAR_CREDENTIAL_HEADER_RE.test(n); +} + +function slimHarHeaders(headers: unknown): any[] { + if (!Array.isArray(headers)) return []; + return headers.filter((h: any) => h && typeof h.name === 'string' && !isHarCredentialHeader(h.name)); +} + +function slimHarQueryString(qs: unknown): any[] { + if (!Array.isArray(qs)) return []; + return qs.filter((q: any) => q && typeof q.name === 'string' && !HAR_CREDENTIAL_HEADER_RE.test(q.name)); +} + +/** `{"user":{"password":"…"}}` → the same shape with every credential-named key's value removed, at any depth. */ +function scrubJsonCredentials(value: any, depth = 0): any { + if (depth > 10 || value === null || typeof value !== 'object') return value; + if (Array.isArray(value)) return value.map((v) => scrubJsonCredentials(v, depth + 1)); + const out: Record = {}; + for (const [key, v] of Object.entries(value)) { + out[key] = HAR_CREDENTIAL_HEADER_RE.test(key) ? '[removed before upload]' : scrubJsonCredentials(v, depth + 1); + } + return out; +} + +function slimHarPostData(postData: unknown): any { + if (!postData || typeof postData !== 'object') return postData; + const pd = postData as any; + if (Array.isArray(pd.params)) { + return { ...pd, params: pd.params.filter((p: any) => !(p && typeof p.name === 'string' && HAR_CREDENTIAL_HEADER_RE.test(p.name))) }; + } + if (typeof pd.text === 'string' && typeof pd.mimeType === 'string' && pd.mimeType.includes('application/json')) { + try { + const parsed = JSON.parse(pd.text); + if (parsed && typeof parsed === 'object') { + return { ...pd, text: JSON.stringify(scrubJsonCredentials(parsed)) }; + } + } catch { + // Not parseable JSON — leave it for the backend, which drops anything it can't scrub. + } + } + return pd; +} + +/** Same origin as the connector's base URL, treating `wss:`/`ws:` as `https:`/`http:` like the backend parser does. */ +function matchesHarOrigin(urlStr: string, base: URL | null): boolean { + if (!base) return true; + try { + const u = new URL(urlStr); + const scheme = (p: string) => (p === 'wss:' ? 'https:' : p === 'ws:' ? 'http:' : p); + return scheme(u.protocol) === scheme(base.protocol) && u.host === base.host; + } catch { + return false; + } +} + +/** + * Strips response data, cookies and credential headers/query/body fields + * from a HAR before it ever leaves the browser, and drops entries outside + * the connector's own origin — a raw capture carries full response bodies, + * session cookies and requests to other sites that must never be uploaded at + * all, on top of the credential fields the backend parser also scrubs. + */ +function slimHarForUpload(rawText: string, baseUrl?: string): { json: string; byteLength: number; droppedOtherOrigin: number } { + const har = JSON.parse(rawText); + const entries = har?.log?.entries; + if (!Array.isArray(entries)) { + throw new Error('Not a valid HAR file (expected a log.entries array)'); + } + let base: URL | null = null; + try { + base = baseUrl ? new URL(baseUrl) : null; + } catch { + base = null; + } + const inScope = entries.filter((entry: any) => matchesHarOrigin(entry?.request?.url ?? '', base)); + const slimmed = { + log: { + version: har.log.version, + entries: inScope.map((entry: any) => ({ + _resourceType: entry?._resourceType, + request: entry?.request + ? { + method: entry.request.method, + url: entry.request.url, + httpVersion: entry.request.httpVersion, + headers: slimHarHeaders(entry.request.headers), + queryString: slimHarQueryString(entry.request.queryString), + postData: slimHarPostData(entry.request.postData), + } + : undefined, + // The backend never reads response data — dropped entirely, not just its body. + })), + }, + }; + const json = JSON.stringify(slimmed); + return { + json, + byteLength: new TextEncoder().encode(json).length, + droppedOtherOrigin: entries.length - inScope.length, + }; +} + export default function ConnectorDetailPage() { const { token } = useAuth(); const params = useParams(); @@ -162,6 +272,11 @@ export default function ConnectorDetailPage() { const [importUrl, setImportUrl] = useState(''); const [importing, setImporting] = useState(false); + // HAR import: the file is slimmed (response bodies, cookies, credential + // headers stripped) in the browser before its JSON lands in importContent. + const [harFileName, setHarFileName] = useState(''); + const [harError, setHarError] = useState(''); + // MCP assign modal — shown when tools are added and connector is not yet assigned const [showMcpAssign, setShowMcpAssign] = useState(false); @@ -510,6 +625,33 @@ export default function ConnectorDetailPage() { } }; + const handleHarFileChange = async (file: File | null) => { + setHarError(''); + setImportContent(''); + setHarFileName(''); + if (!file) return; + setHarFileName(file.name); + try { + const text = await file.text(); + const { json, byteLength, droppedOtherOrigin } = slimHarForUpload(text, connector?.baseUrl); + if (byteLength > HAR_MAX_UPLOAD_BYTES) { + setHarError( + `This capture is still ${formatBytes(byteLength)} after removing response bodies and cookies, which is over what the server accepts. Re-capture a smaller session (fewer requests) and try again.`, + ); + return; + } + if (droppedOtherOrigin > 0 && JSON.parse(json).log.entries.length === 0) { + setHarError( + `None of the ${droppedOtherOrigin} request(s) in this capture were made to ${connector?.baseUrl ?? 'the connector\'s base URL'}. Capture a session against that host and try again.`, + ); + return; + } + setImportContent(json); + } catch (err: any) { + setHarError(`Could not read this file: ${err.message}`); + } + }; + const handleImportTools = async () => { if (!token) return; setImporting(true); @@ -522,7 +664,7 @@ export default function ConnectorDetailPage() { if (importContent.trim()) data.content = importContent; const sets = importEntitySets.split(/[\s,]+/).map((x) => x.trim()).filter(Boolean); if (sets.length) data.entitySets = sets; - } else if (importSource === 'curl' || importSource === 'json') { + } else if (importSource === 'curl' || importSource === 'json' || importSource === 'har') { data.content = importContent; } else if (importUrl) { data.url = importUrl; @@ -537,6 +679,8 @@ export default function ConnectorDetailPage() { setShowImport(false); setImportContent(''); setImportUrl(''); + setHarFileName(''); + setHarError(''); fetchConnector(); promptMcpAssignIfNeeded(); } @@ -1672,7 +1816,7 @@ export default function ConnectorDetailPage() { {IMPORT_SOURCES.filter((src) => src.id !== 'odata' || hasOData(connector)).map((src) => (