From 2e9cadc8833c5220ad9ace69a38f81bf02dae6c9 Mon Sep 17 00:00:00 2001 From: David Tippett Date: Thu, 23 Jul 2026 20:26:08 -0400 Subject: [PATCH 1/2] Add --valuable-result-positions to search feedback Wire the API's new valuableResultPositions field through both feedback commands so agents can attribute usefulness to specific data.web results by 1-indexed position. Skills now surface positions in jq extraction snippets and require exhaustive position marking (unlisted results count as not useful), with valuableSources reserved for URLs outside data.web. Requires an API with position-based search feedback support (firecrawl/firecrawl#4109); the field is only sent when the flag is provided. Co-Authored-By: Claude Fable 5 --- README.md | 29 +++++++++++++++-------------- skills/firecrawl-cli/SKILL.md | 16 +++++++--------- skills/firecrawl-search/SKILL.md | 11 ++++++----- src/commands/feedback.ts | 27 +++++++++++++++++++++++---- src/commands/search-feedback.ts | 7 +++++++ src/index.ts | 30 +++++++++++++++++++++++++++++- 6 files changed, 87 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 4c2c637570..654219b914 100644 --- a/README.md +++ b/README.md @@ -370,20 +370,21 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| --------------------------------------------- | ------------------------------------------------------------------ | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--valuable-result-positions ` | Search only: 1-indexed `data.web` positions of every useful result | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index 5494c1e3cb..ae17fce5f5 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -193,9 +193,7 @@ The `check` response then carries a per-field diff (paths like `plans[0].price`) }, "snapshot": { "json": { - "plans": [ - /* current full extraction */ - ] + "plans": [/* current full extraction */] } } } @@ -255,11 +253,11 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li These patterns are useful when working with file-based output (`-o` flag) for complex tasks: ```bash -# Extract URLs from search -jq -r '.data.web[].url' .firecrawl/search.json +# Extract URLs from search with their 1-indexed positions (needed for feedback) +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json -# Get titles and URLs -jq -r '.data.web[] | "\(.title): \(.url)"' .firecrawl/search.json +# Get positions, titles, and URLs +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json ``` ## After search: send feedback (refunds 1 credit) @@ -271,13 +269,13 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ --query-suggestions "Boost react.dev for react-hooks queries" \ --silent & ``` -The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. +The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-result-positions` (1-indexed into `data.web`, list **every** useful one — unlisted results count as not useful); reserve `--valuable-sources` for useful URLs that were not in `data.web`. **Opt out:** `export FIRECRAWL_NO_SEARCH_FEEDBACK=1` makes the CLI skip every feedback call silently. Respect that flag — do not try to work around it. See [firecrawl-search](../firecrawl-search/SKILL.md) for the full pattern. diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 87b426cf92..d8437f5c90 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -49,7 +49,7 @@ firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json - **`--scrape` fetches full content** — don't re-scrape URLs from search results. This saves credits and avoids redundant fetches. - Always write results to `.firecrawl/` with `-o` to avoid context window bloat. -- Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` +- Use `jq` to extract URLs or titles **with their 1-indexed positions** (you'll need the positions for feedback): `jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` ## Send feedback after every search (refunds 1 credit) @@ -62,9 +62,10 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Time window:** must be sent within ~2 minutes of the search. Late feedback is rejected. - **`--missing-content` is the most important field.** It's a list of _specific pieces_ of content you expected but did not find. One topic per entry — do not pack multiple topics into one string. These aggregate across teams and tell us what to index next. +- **`--valuable-result-positions` marks which results were useful.** 1-indexed positions into `data.web` (web results only). **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT in `data.web` (e.g. a page you found by following a result's link) — never report the same result in both. - **Substantive content required** (zero-effort feedback is rejected with HTTP 400): - - `good` → must include at least one `--valuable-sources` entry. - - `partial` → must include `--valuable-sources` or `--missing-content`. + - `good` → must include `--valuable-result-positions` or at least one `--valuable-sources` entry. + - `partial` → must include `--valuable-result-positions`, `--valuable-sources`, or `--missing-content`. - `bad` → must include `--missing-content` or `--query-suggestions`. - **Daily refund cap (per team, per UTC day, default 100 credits).** Once your team has been refunded 100 credits today, further submissions still record feedback but no longer refund credits. The response includes `creditsRefundedToday` / `dailyRefundCap` / `dailyCapReached`. **When `dailyCapReached: true`, stop calling `search-feedback` for the rest of the UTC day** — it won't refund anything and you're wasting bandwidth. - **Idempotent:** re-submitting for the same search id returns success but no extra refund. @@ -79,10 +80,10 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) Then send feedback. Pick the rating that matches what actually happened: ```bash -# Results were useful, with notes on what was still missing +# Results were useful — positions 1 and 3 of data.web answered the question firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[ {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, {"topic":"useTransition","description":"No coverage of useTransition for routing"} diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 14318a8c9e..e14e643672 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -20,6 +20,7 @@ export interface EndpointFeedbackOptions { tags?: string[]; note?: string; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; url?: string; @@ -111,8 +112,9 @@ export function parseFeedbackListArg( return normalizeList(trimmed.split(',')); } -export function parsePageNumbersArg( - raw: string | undefined +function parsePositiveIntArrayArg( + raw: string | undefined, + flag: string ): number[] | undefined { if (!raw) return undefined; const trimmed = raw.trim(); @@ -123,12 +125,12 @@ export function parsePageNumbersArg( try { const parsed = JSON.parse(trimmed); if (!Array.isArray(parsed)) { - throw new Error('--page-numbers must be a JSON array.'); + throw new Error(`${flag} must be a JSON array.`); } values = parsed; } catch { throw new Error( - '--page-numbers must be a comma-separated list or valid JSON array.' + `${flag} must be a comma-separated list or valid JSON array.` ); } } else { @@ -144,6 +146,18 @@ export function parsePageNumbersArg( return numbers.length > 0 ? numbers : undefined; } +export function parsePageNumbersArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--page-numbers'); +} + +export function parseValuableResultPositionsArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--valuable-result-positions'); +} + export function parseMetadataArg( raw: string | undefined, filePath: string | undefined @@ -207,6 +221,7 @@ export function parseEndpointFeedbackCliOptions(options: { metadata?: string; metadataFile?: string; valuableSources?: string; + valuableResultPositions?: string; missingContent?: string | string[]; rating?: string; }) { @@ -217,6 +232,9 @@ export function parseEndpointFeedbackCliOptions(options: { pageNumbers: parsePageNumbersArg(options.pageNumbers), metadata: parseMetadataArg(options.metadata, options.metadataFile), valuableSources: parseValuableSourcesArg(options.valuableSources), + valuableResultPositions: parseValuableResultPositionsArg( + options.valuableResultPositions + ), missingContent: parseMissingContentArg(options.missingContent), }; } @@ -261,6 +279,7 @@ export async function executeEndpointFeedback( ['tags', normalizeList(options.tags)], ['note', options.note], ['valuableSources', options.valuableSources], + ['valuableResultPositions', options.valuableResultPositions], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], ['url', options.url], diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index 35898e78b4..e5c9d46d2a 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -17,6 +17,7 @@ export interface SearchFeedbackOptions { searchId: string; rating: SearchFeedbackRating; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; apiKey?: string; @@ -118,6 +119,12 @@ export async function executeSearchFeedback( ...(s.reason ? { reason: s.reason } : {}), })); } + if ( + options.valuableResultPositions && + options.valuableResultPositions.length > 0 + ) { + body.valuableResultPositions = options.valuableResultPositions; + } if (options.missingContent && options.missingContent.length > 0) { body.missingContent = options.missingContent .filter((m) => !!m.topic) diff --git a/src/index.ts b/src/index.ts index 5c1ae11f3f..a5069533d8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,6 +37,7 @@ import { handleEndpointFeedbackCommand, parseEndpointFeedbackCliOptions, parseEndpointFeedbackEndpoint, + parseValuableResultPositionsArg, } from './commands/feedback'; import { handleAgentCommand } from './commands/agent'; import { @@ -1294,9 +1295,16 @@ function createSearchFeedbackCommand(): Command { ) .argument('', 'The id returned by `firecrawl search ... --json`') .requiredOption('--rating ', 'Overall rating: good | bad | partial') + .option( + '--valuable-result-positions ', + '1-indexed positions in data.web of every result that was useful ' + + '(e.g. "1,3" or [1,3]). Unlisted results are treated as not useful.' + ) .option( '--valuable-sources ', - 'Comma-separated URLs OR JSON array of {url, reason} entries' + 'Comma-separated URLs OR JSON array of {url, reason} entries. ' + + 'For useful URLs NOT in data.web; use --valuable-result-positions ' + + 'for returned web results.' ) .option( '--missing-content ', @@ -1337,6 +1345,19 @@ function createSearchFeedbackCommand(): Command { process.exit(1); } + let valuableResultPositions; + try { + valuableResultPositions = parseValuableResultPositionsArg( + options.valuableResultPositions + ); + } catch (error: any) { + console.error( + 'Error:', + error?.message || 'Invalid --valuable-result-positions' + ); + process.exit(1); + } + let missingContent; try { missingContent = parseMissingContentArg(options.missingContent); @@ -1349,6 +1370,7 @@ function createSearchFeedbackCommand(): Command { searchId, rating: rating as SearchFeedbackRating, valuableSources, + valuableResultPositions, missingContent, querySuggestions: options.querySuggestions, apiKey: options.apiKey, @@ -1385,6 +1407,11 @@ function createFeedbackCommand(): Command { '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' ) + .option( + '--valuable-result-positions ', + 'Search only: 1-indexed positions in data.web of every useful result ' + + '(e.g. "1,3" or [1,3])' + ) .option( '--missing-content ', 'Specific pieces of content missing from results. ' + @@ -1440,6 +1467,7 @@ function createFeedbackCommand(): Command { tags: parsed.tags, note: options.note, valuableSources: parsed.valuableSources, + valuableResultPositions: parsed.valuableResultPositions, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, url: options.url, From fb87413bae93c69771a5716b15ac7057ab989a6a Mon Sep 17 00:00:00 2001 From: David Tippett Date: Wed, 12 Aug 2026 16:10:19 -0400 Subject: [PATCH 2/2] Replace --valuable-result-positions with --valuable-results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search results come back grouped — data.web, data.images, data.news — and each group is numbered from 1 independently, so a bare position only ever meant "web" and could not name a news or image result at all. That matters most for those groups: news and image results have optional URLs, so --valuable-sources cannot reliably address them either. Takes "source:position" pairs (e.g. "web:1,news:2") or a JSON array of {source, position, reason} entries, matching the API's valuableResults field (firecrawl/firecrawl#4109). The source is always required — "web:1" and "news:1" are different results. Also reverts the generic parsePositiveIntArrayArg helper this branch had extracted from parsePageNumbersArg; nothing shares it now. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 30 +++---- skills/firecrawl-cli/SKILL.md | 13 +-- skills/firecrawl-search/SKILL.md | 12 +-- src/__tests__/commands/feedback.test.ts | 50 +++++++++++ src/commands/feedback.ts | 33 +++---- src/commands/search-feedback.ts | 115 ++++++++++++++++++++++-- src/index.ts | 35 ++++---- 7 files changed, 213 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 654219b914..ea64d22dae 100644 --- a/README.md +++ b/README.md @@ -370,21 +370,21 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| --------------------------------------------- | ------------------------------------------------------------------ | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--valuable-result-positions ` | Search only: 1-indexed `data.web` positions of every useful result | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| -------------------------------------- | -------------------------------------------------------------------------- | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--valuable-results ` | Search only: every useful result as `source:position`, e.g. `web:1,news:2` | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index ae17fce5f5..a51b2a7eac 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -253,11 +253,12 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li These patterns are useful when working with file-based output (`-o` flag) for complex tasks: ```bash -# Extract URLs from search with their 1-indexed positions (needed for feedback) -jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json +# Extract URLs with their source and 1-indexed position (needed for feedback). +# Each group is numbered from 1 independently, so keep the source with it. +jq -r '.data | to_entries[] | .key as $s | .value | to_entries[] | "\($s):\(.key + 1)\t\(.value.url)"' .firecrawl/search.json -# Get positions, titles, and URLs -jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json +# Web results only, with positions, titles, and URLs +jq -r '.data.web | to_entries[] | "web:\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json ``` ## After search: send feedback (refunds 1 credit) @@ -269,13 +270,13 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-result-positions "1,3" \ + --valuable-results "web:1,web:3" \ --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ --query-suggestions "Boost react.dev for react-hooks queries" \ --silent & ``` -The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-result-positions` (1-indexed into `data.web`, list **every** useful one — unlisted results count as not useful); reserve `--valuable-sources` for useful URLs that were not in `data.web`. +The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-results` as `source:position` (e.g. `web:1,news:2`) — results come back grouped and each group is numbered from 1, so the source is required. List **every** useful one; unlisted results count as not useful. Reserve `--valuable-sources` for useful URLs that were not among the returned results. **Opt out:** `export FIRECRAWL_NO_SEARCH_FEEDBACK=1` makes the CLI skip every feedback call silently. Respect that flag — do not try to work around it. See [firecrawl-search](../firecrawl-search/SKILL.md) for the full pattern. diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index d8437f5c90..907706e55b 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -49,7 +49,7 @@ firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json - **`--scrape` fetches full content** — don't re-scrape URLs from search results. This saves credits and avoids redundant fetches. - Always write results to `.firecrawl/` with `-o` to avoid context window bloat. -- Use `jq` to extract URLs or titles **with their 1-indexed positions** (you'll need the positions for feedback): `jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` +- Use `jq` to extract URLs **with their source and 1-indexed position** (you'll need both for feedback): `jq -r '.data | to_entries[] | .key as $s | .value | to_entries[] | "\($s):\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` ## Send feedback after every search (refunds 1 credit) @@ -62,10 +62,10 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Time window:** must be sent within ~2 minutes of the search. Late feedback is rejected. - **`--missing-content` is the most important field.** It's a list of _specific pieces_ of content you expected but did not find. One topic per entry — do not pack multiple topics into one string. These aggregate across teams and tell us what to index next. -- **`--valuable-result-positions` marks which results were useful.** 1-indexed positions into `data.web` (web results only). **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT in `data.web` (e.g. a page you found by following a result's link) — never report the same result in both. +- **`--valuable-results` marks which results were useful.** Results come back grouped (`data.web`, `data.images`, `data.news`) and **each group is numbered from 1 independently**, so every entry is `source:position` — `web:1` and `news:1` are two different results. **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT among the returned results (e.g. a page you found by following a result's link) — never report the same result in both. - **Substantive content required** (zero-effort feedback is rejected with HTTP 400): - - `good` → must include `--valuable-result-positions` or at least one `--valuable-sources` entry. - - `partial` → must include `--valuable-result-positions`, `--valuable-sources`, or `--missing-content`. + - `good` → must include `--valuable-results` or at least one `--valuable-sources` entry. + - `partial` → must include `--valuable-results`, `--valuable-sources`, or `--missing-content`. - `bad` → must include `--missing-content` or `--query-suggestions`. - **Daily refund cap (per team, per UTC day, default 100 credits).** Once your team has been refunded 100 credits today, further submissions still record feedback but no longer refund credits. The response includes `creditsRefundedToday` / `dailyRefundCap` / `dailyCapReached`. **When `dailyCapReached: true`, stop calling `search-feedback` for the rest of the UTC day** — it won't refund anything and you're wasting bandwidth. - **Idempotent:** re-submitting for the same search id returns success but no extra refund. @@ -80,10 +80,10 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) Then send feedback. Pick the rating that matches what actually happened: ```bash -# Results were useful — positions 1 and 3 of data.web answered the question +# Results were useful — web positions 1 and 3 answered the question firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-result-positions "1,3" \ + --valuable-results "web:1,web:3" \ --missing-content '[ {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, {"topic":"useTransition","description":"No coverage of useTransition for routing"} diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index cbb906ace7..e152a284c6 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -6,6 +6,7 @@ import { parseFeedbackListArg, parsePageNumbersArg, } from '../../commands/feedback'; +import { parseValuableResultsArg } from '../../commands/search-feedback'; import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; @@ -207,4 +208,53 @@ describe('feedback parsing', () => { expect(parsePageNumbersArg('1, 2, bad, -1, 3')).toEqual([1, 2, 3]); expect(parsePageNumbersArg('[4,5]')).toEqual([4, 5]); }); + + it('parses valuable results as source:position pairs', () => { + expect(parseValuableResultsArg('web:1, news:2')).toEqual([ + { source: 'web', position: 1 }, + { source: 'news', position: 2 }, + ]); + expect(parseValuableResultsArg('images:3')).toEqual([ + { source: 'images', position: 3 }, + ]); + }); + + it('parses valuable results from JSON, keeping reasons', () => { + expect( + parseValuableResultsArg( + '[{"source":"web","position":1,"reason":"Answered it"},{"source":"news","position":2}]' + ) + ).toEqual([ + { source: 'web', position: 1, reason: 'Answered it' }, + { source: 'news', position: 2 }, + ]); + }); + + // Each group is numbered from 1 independently, so a bare position does not + // identify a result. + it('rejects valuable results without a source', () => { + expect(() => parseValuableResultsArg('1,3')).toThrow( + 'must be "source:position"' + ); + expect(() => parseValuableResultsArg('[{"position":1}]')).toThrow( + 'source must be one of' + ); + }); + + it('rejects unknown sources and non-positive positions', () => { + expect(() => parseValuableResultsArg('video:1')).toThrow( + 'source must be one of' + ); + expect(() => parseValuableResultsArg('web:0')).toThrow( + 'positions must be integers of 1 or greater' + ); + expect(() => parseValuableResultsArg('web:abc')).toThrow( + 'positions must be integers of 1 or greater' + ); + }); + + it('returns undefined for empty input', () => { + expect(parseValuableResultsArg(undefined)).toBeUndefined(); + expect(parseValuableResultsArg(' ')).toBeUndefined(); + }); }); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index e14e643672..96b48f01f9 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -4,9 +4,11 @@ import { getConfig, isCustomApiUrl, validateConfig } from '../utils/config'; import { getClient } from '../utils/client'; import { parseMissingContentArg, + parseValuableResultsArg, parseValuableSourcesArg, type MissingContentInput, type SearchFeedbackRating, + type ValuableResultInput, type ValuableSourceInput, } from './search-feedback'; @@ -20,7 +22,7 @@ export interface EndpointFeedbackOptions { tags?: string[]; note?: string; valuableSources?: ValuableSourceInput[]; - valuableResultPositions?: number[]; + valuableResults?: ValuableResultInput[]; missingContent?: MissingContentInput[]; querySuggestions?: string; url?: string; @@ -112,9 +114,8 @@ export function parseFeedbackListArg( return normalizeList(trimmed.split(',')); } -function parsePositiveIntArrayArg( - raw: string | undefined, - flag: string +export function parsePageNumbersArg( + raw: string | undefined ): number[] | undefined { if (!raw) return undefined; const trimmed = raw.trim(); @@ -125,12 +126,12 @@ function parsePositiveIntArrayArg( try { const parsed = JSON.parse(trimmed); if (!Array.isArray(parsed)) { - throw new Error(`${flag} must be a JSON array.`); + throw new Error('--page-numbers must be a JSON array.'); } values = parsed; } catch { throw new Error( - `${flag} must be a comma-separated list or valid JSON array.` + '--page-numbers must be a comma-separated list or valid JSON array.' ); } } else { @@ -146,18 +147,6 @@ function parsePositiveIntArrayArg( return numbers.length > 0 ? numbers : undefined; } -export function parsePageNumbersArg( - raw: string | undefined -): number[] | undefined { - return parsePositiveIntArrayArg(raw, '--page-numbers'); -} - -export function parseValuableResultPositionsArg( - raw: string | undefined -): number[] | undefined { - return parsePositiveIntArrayArg(raw, '--valuable-result-positions'); -} - export function parseMetadataArg( raw: string | undefined, filePath: string | undefined @@ -221,7 +210,7 @@ export function parseEndpointFeedbackCliOptions(options: { metadata?: string; metadataFile?: string; valuableSources?: string; - valuableResultPositions?: string; + valuableResults?: string; missingContent?: string | string[]; rating?: string; }) { @@ -232,9 +221,7 @@ export function parseEndpointFeedbackCliOptions(options: { pageNumbers: parsePageNumbersArg(options.pageNumbers), metadata: parseMetadataArg(options.metadata, options.metadataFile), valuableSources: parseValuableSourcesArg(options.valuableSources), - valuableResultPositions: parseValuableResultPositionsArg( - options.valuableResultPositions - ), + valuableResults: parseValuableResultsArg(options.valuableResults), missingContent: parseMissingContentArg(options.missingContent), }; } @@ -279,7 +266,7 @@ export async function executeEndpointFeedback( ['tags', normalizeList(options.tags)], ['note', options.note], ['valuableSources', options.valuableSources], - ['valuableResultPositions', options.valuableResultPositions], + ['valuableResults', options.valuableResults], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], ['url', options.url], diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index e5c9d46d2a..f99994766a 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -13,11 +13,28 @@ export interface MissingContentInput { description?: string; } +export type SearchResultSource = 'web' | 'images' | 'news'; + +export const SEARCH_RESULT_SOURCES: readonly SearchResultSource[] = [ + 'web', + 'images', + 'news', +]; + +// Search results come back grouped — data.web, data.images, data.news — and +// each group is numbered from 1 independently, so a position is only +// meaningful alongside the group it indexes into. +export interface ValuableResultInput { + source: SearchResultSource; + position: number; + reason?: string; +} + export interface SearchFeedbackOptions { searchId: string; rating: SearchFeedbackRating; valuableSources?: ValuableSourceInput[]; - valuableResultPositions?: number[]; + valuableResults?: ValuableResultInput[]; missingContent?: MissingContentInput[]; querySuggestions?: string; apiKey?: string; @@ -119,11 +136,8 @@ export async function executeSearchFeedback( ...(s.reason ? { reason: s.reason } : {}), })); } - if ( - options.valuableResultPositions && - options.valuableResultPositions.length > 0 - ) { - body.valuableResultPositions = options.valuableResultPositions; + if (options.valuableResults && options.valuableResults.length > 0) { + body.valuableResults = options.valuableResults; } if (options.missingContent && options.missingContent.length > 0) { body.missingContent = options.missingContent @@ -357,6 +371,95 @@ export function parseValuableSourcesArg( .map((url) => ({ url })); } +function isSearchResultSource(value: unknown): value is SearchResultSource { + return ( + typeof value === 'string' && + (SEARCH_RESULT_SOURCES as readonly string[]).includes(value) + ); +} + +function parsePositionValue(raw: unknown, flag: string): number { + const position = typeof raw === 'string' ? Number(raw.trim()) : raw; + if ( + typeof position !== 'number' || + !Number.isInteger(position) || + position < 1 + ) { + throw new Error(`${flag} positions must be integers of 1 or greater.`); + } + return position; +} + +// Accepts a compact "source:position" list (e.g. "web:1,news:2") or a JSON +// array of {source, position, reason} entries. The source is always required: +// results are grouped and each group is numbered from 1, so a bare position +// does not identify a result. +export function parseValuableResultsArg( + raw: string | undefined, + flag = '--valuable-results' +): ValuableResultInput[] | undefined { + if (!raw) return undefined; + const trimmed = raw.trim(); + if (!trimmed) return undefined; + + const sourceList = SEARCH_RESULT_SOURCES.join(' | '); + + if (trimmed.startsWith('[') || trimmed.startsWith('{')) { + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new Error( + `${flag} must be valid JSON or a comma-separated "source:position" list.` + ); + } + + const entries = Array.isArray(parsed) ? parsed : [parsed]; + const cleaned = entries.map((entry: any) => { + if (!entry || typeof entry !== 'object') { + throw new Error( + `${flag} JSON entries must be objects with a source and a position.` + ); + } + if (!isSearchResultSource(entry.source)) { + throw new Error(`${flag} source must be one of: ${sourceList}.`); + } + return { + source: entry.source, + position: parsePositionValue(entry.position, flag), + ...(typeof entry.reason === 'string' && entry.reason.trim() + ? { reason: entry.reason } + : {}), + }; + }); + return cleaned.length > 0 ? cleaned : undefined; + } + + const cleaned = trimmed + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => { + const separator = entry.lastIndexOf(':'); + if (separator === -1) { + throw new Error( + `${flag} entries must be "source:position" (e.g. web:1) — ` + + `results are grouped, so a bare position is ambiguous.` + ); + } + const source = entry.slice(0, separator).trim(); + if (!isSearchResultSource(source)) { + throw new Error(`${flag} source must be one of: ${sourceList}.`); + } + return { + source, + position: parsePositionValue(entry.slice(separator + 1), flag), + }; + }); + + return cleaned.length > 0 ? cleaned : undefined; +} + // Accepts JSON arrays/objects, "topic: description" strings, comma- // separated topic lists, or repeated values. Caps at 20 entries. export function parseMissingContentArg( diff --git a/src/index.ts b/src/index.ts index a5069533d8..2983dc76d9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,7 @@ import { import { handleSearchFeedbackCommand, parseValuableSourcesArg, + parseValuableResultsArg, parseMissingContentArg, type SearchFeedbackRating, } from './commands/search-feedback'; @@ -37,7 +38,6 @@ import { handleEndpointFeedbackCommand, parseEndpointFeedbackCliOptions, parseEndpointFeedbackEndpoint, - parseValuableResultPositionsArg, } from './commands/feedback'; import { handleAgentCommand } from './commands/agent'; import { @@ -1296,15 +1296,17 @@ function createSearchFeedbackCommand(): Command { .argument('', 'The id returned by `firecrawl search ... --json`') .requiredOption('--rating ', 'Overall rating: good | bad | partial') .option( - '--valuable-result-positions ', - '1-indexed positions in data.web of every result that was useful ' + - '(e.g. "1,3" or [1,3]). Unlisted results are treated as not useful.' + '--valuable-results ', + 'Every result that was useful, as "source:position" (e.g. ' + + '"web:1,news:2") OR a JSON array of {source, position, reason}. ' + + 'Results are grouped and each group is numbered from 1, so the ' + + 'source is required. Unlisted results are treated as not useful.' ) .option( '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries. ' + - 'For useful URLs NOT in data.web; use --valuable-result-positions ' + - 'for returned web results.' + 'For useful URLs NOT among the returned results; use ' + + '--valuable-results for results the search returned.' ) .option( '--missing-content ', @@ -1345,16 +1347,11 @@ function createSearchFeedbackCommand(): Command { process.exit(1); } - let valuableResultPositions; + let valuableResults; try { - valuableResultPositions = parseValuableResultPositionsArg( - options.valuableResultPositions - ); + valuableResults = parseValuableResultsArg(options.valuableResults); } catch (error: any) { - console.error( - 'Error:', - error?.message || 'Invalid --valuable-result-positions' - ); + console.error('Error:', error?.message || 'Invalid --valuable-results'); process.exit(1); } @@ -1370,7 +1367,7 @@ function createSearchFeedbackCommand(): Command { searchId, rating: rating as SearchFeedbackRating, valuableSources, - valuableResultPositions, + valuableResults, missingContent, querySuggestions: options.querySuggestions, apiKey: options.apiKey, @@ -1408,9 +1405,9 @@ function createFeedbackCommand(): Command { 'Comma-separated URLs OR JSON array of {url, reason} entries' ) .option( - '--valuable-result-positions ', - 'Search only: 1-indexed positions in data.web of every useful result ' + - '(e.g. "1,3" or [1,3])' + '--valuable-results ', + 'Search only: every useful result as "source:position" (e.g. ' + + '"web:1,news:2") OR a JSON array of {source, position, reason}' ) .option( '--missing-content ', @@ -1467,7 +1464,7 @@ function createFeedbackCommand(): Command { tags: parsed.tags, note: options.note, valuableSources: parsed.valuableSources, - valuableResultPositions: parsed.valuableResultPositions, + valuableResults: parsed.valuableResults, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, url: options.url,