Skip to content

fix(crawl): re-anchor same-origin scope to the seed's post-redirect origin #265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
divya0795 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(crawl): re-anchor same-origin scope to the seed's post-redirect origin #265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/crawl/crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,12 @@ export class Crawler {
const seenEdges = new Set<string>();
const indexing = isIndexingEnabled();

// seedOrigin comes from the pre-redirect input URL. Re-anchored to the
// seed's final origin below once it has been fetched, so a seed that
// redirects (apex -> www, http -> https) doesn't reject every link on the
// page it actually landed on.
let scopeOrigin = seedOrigin;

// Queue: [url, depth]
const queue: Array<[string, number]> = [[input.url, 0]];
visited.add(canonicalForCrawl(input.url));
Expand Down Expand Up @@ -132,6 +138,15 @@ export class Crawler {
continue;
}

// depth 0 is only ever the seed: scope the crawl to where it landed.
if (depth === 0) {
try {
scopeOrigin = new URL(fetchResult.url).origin;
} catch {

This comment was marked as spam.

Sign in to view

// unparseable final URL — keep the origin we started with
}
}

Comment on lines +141 to +149

@coderabbitai coderabbitai Bot Aug 1, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔒 Security & Privacy | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Expect: FetchOutput producers validate final URLs as HTTP(S).
rg -n -C 8 \
  'interface FetchOutput|type FetchOutput|FetchOutput|response\.url|fetchResult\.url' \
  src tests || true

Repository: KnockOutEZ/wigolo

Length of output: 50373

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== crawler relevant sections =="
sed -n '120,185p' src/crawl/crawler.ts
sed -n '1,90p' src/crawl/crawler.ts

echo "== URL validators/link extraction =="
sed -n '45,150p' src/tools/fetch.ts
rg -n "isAbsoluteUrl|http(s)?:|finalUrl|response\.url|browser|rawFetch|parseLinks|extractLinks" src tests/unit/crawl/crawler.test.ts -A3 -B3

echo "== behavior prototype with JS URL =="
node - <<'JS'
for (const u of ['https://example.com', 'http://example.com', 'file:///tmp/x.html', 'data:text/plain;base64,abc', 'javascript:alert(1)']) {
  const url = new URL(u)
  console.log(JSON.stringify({ url, origin: url.origin }))
}
JS

Repository: KnockOutEZ/wigolo

Length of output: 50374

🏁 Script executed:

#!/bin/bash
set -euo pipefail

LOG=/tmp/coderabbit-shell-logs/shell-output-ZV3CUR

echo "== exact crawler sections =="
python3 - <<'PY'
from pathlib import Path
p=Path('src/crawl/crawler.ts')
lines=p.read_text().splitlines()
for start,end in [(1,90),(120,185)]:
    print(f'\n--- src/crawl/crawler.ts {start}-{end} ---')
    for i in range(start,end+1):
        print(f'{i}: {lines[i-1]}')
PY

echo "== crawl helper validation and input schema =="
sed -n '1,95p' src/tools/crawl.ts
sed -n '180,235p' src/tools/crawl.ts
sed -n '190,235p' src/server/tool-schemas.ts
sed -n '85,180p' src/watch/ssrf.ts

echo "== focused router/validator references =="
rg -n "isAbsoluteUrl|safeAbsoluteUrl|fetch\(url|finalUrl|redirect|location|response\.url|browser|playwright|navigate|waitForNavigation" src -S --glob '!**/*.md' | head -n 160

Repository: KnockOutEZ/wigolo

Length of output: 33527

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== src/crawl/crawler.ts traversal implementation =="
python3 - <<'PY'
from pathlib import Path
p=Path('src/crawl/crawler.ts')
lines=p.read_text().splitlines()
for start,end in [(90,120),(230,360)]:
    print(f'\n--- src/crawl/crawler.ts {start}-{end} ---')
    for i in range(start,end+1):
        print(f'{i}: {lines[i-1]}')
PY

echo "== browser fetch/finalUrl paths =="
sed -n '656,730p' src/fetch/router.ts
rg -n "finalUrl|currentUrl|get().url|waitForLoadState|page\.url|response\.url|navigationUrl|goto\\(" src/fetch -S --glob '*.ts' | head -n 200

echo "== Node URL behavior =="
node - <<'JS'
for (const u of ['https://example.com', 'http://example.com', 'file:///tmp/x.html', 'data:text/plain;base64,abc', 'javascript:alert(1)']) {
  const url = new URL(u)
  console.log(JSON.stringify({ url, protocol: url.protocol, origin: url.origin }))
}
JS

Repository: KnockOutEZ/wigolo

Length of output: 14891

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== filterLinks implementation =="
rg -n "function filterLinks|filterLinks\\(" src/crawl/crawler.ts src/crawl/url-utils.ts -A 120 -B 20

echo "== crawler link canonicalizers =="
sed -n '1,120p' src/crawl/url-utils.ts

echo "== browser pool navigation/finalUrl around guard/follow =="
sed -n '542,630p' src/fetch/browser-pool.ts

Repository: KnockOutEZ/wigolo

Length of output: 188

Reject opaque-final origins before reuse.

If FetchOutput.url contains file:, data:, javascript:, or another non-HTTP(S) scheme, new URL(...).origin returns "null" instead of throwing. That makes file: or data: links from the fetched page pass scopeOrigin checks and be queued. Set scopeOrigin only after parsing fetchResult.url as http:/https:, or enforce those schemes in the fetch tier before FetchOutput.url is returned.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/crawl/crawler.ts` around lines 141 - 149, Update the depth-0 scopeOrigin
assignment around fetchResult.url so the parsed URL is accepted only when its
protocol is http: or https:. Leave scopeOrigin unchanged for opaque or other
schemes, including cases where URL parsing succeeds but returns origin "null",
while preserving the existing fallback for parse failures.

const item: CrawlResultItem = {
url: canonicalForOutput(fetchResult.url),
title: fetchResult.title,
Expand All @@ -147,7 +162,7 @@ export class Crawler {

// Discover links for traversal
if (depth < maxDepth) {
const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
const newLinks = this.filterLinks(fetchResult.links, scopeOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);

// filterLinks() runs against the visited snapshot before this loop,
// so two outbound links with the same canonical (e.g. /page#a and
Expand Down
60 changes: 60 additions & 0 deletions tests/unit/crawl/crawler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,66 @@ describe('Crawler — BFS', () => {
expect(urls).not.toContain('https://other.example.com/external');
});

describe('seed redirect', () => {
// https://example.com 301s to https://www.example.com, so every link on the
// page we actually landed on carries the www host.
const redirectingFetch: FetchFn = vi.fn(async (url: string) => {
if (url === 'https://example.com') {
return makeFetchOutput('https://www.example.com', 'Home', '# Home', [
'https://www.example.com/a',
'https://www.example.com/b',
'https://other.example.com/external',
]);
}
return makeFetchOutput(url, 'Page', '# Page', []);
});

it('scopes the crawl to the seed final origin, not the requested one', async () => {
const crawler = new Crawler(redirectingFetch, rawFetchFn);
const result = await crawler.crawl({
url: 'https://example.com',
strategy: 'bfs',
max_depth: 1,
max_pages: 10,
});

const urls = result.pages.map((p) => p.url);
expect(urls).toContain('https://www.example.com/a');
expect(urls).toContain('https://www.example.com/b');
expect(result.pages.length).toBe(3);
});

it('still rejects off-origin links after re-anchoring', async () => {
const crawler = new Crawler(redirectingFetch, rawFetchFn);
const result = await crawler.crawl({
url: 'https://example.com',
strategy: 'bfs',
max_depth: 1,
max_pages: 10,
});

expect(result.pages.map((p) => p.url)).not.toContain('https://other.example.com/external');
});

it('keeps the requested origin when the seed fetch fails', async () => {
const failingSeed: FetchFn = vi.fn(async (url: string) => {
if (url === 'https://example.com') {
return { ...makeFetchOutput(url, '', '', []), error: 'fetch_failed' };
}
return makeFetchOutput(url, 'Page', '# Page', []);
});
const crawler = new Crawler(failingSeed, rawFetchFn);
const result = await crawler.crawl({
url: 'https://example.com',
strategy: 'bfs',
max_depth: 1,
max_pages: 10,
});

expect(result.pages).toEqual([]);
});
});

it('does not visit the same URL twice', async () => {
const crawler = new Crawler(fetchFn, rawFetchFn);
await crawler.crawl({
Expand Down