diff --git a/src/crawl/crawler.ts b/src/crawl/crawler.ts index ec6582f2f..5fdcd82b0 100644 --- a/src/crawl/crawler.ts +++ b/src/crawl/crawler.ts @@ -99,6 +99,12 @@ export class Crawler { const seenEdges = new Set(); const indexing = isIndexingEnabled(); + // seedOrigin comes from the pre-redirect input URL. Re-anchored to the + // seed's final origin below once it has been fetched, so a seed that + // redirects (apex -> www, http -> https) doesn't reject every link on the + // page it actually landed on. + let scopeOrigin = seedOrigin; + // Queue: [url, depth] const queue: Array<[string, number]> = [[input.url, 0]]; visited.add(canonicalForCrawl(input.url)); @@ -132,6 +138,15 @@ export class Crawler { continue; } + // depth 0 is only ever the seed: scope the crawl to where it landed. + if (depth === 0) { + try { + scopeOrigin = new URL(fetchResult.url).origin; + } catch { + // unparseable final URL — keep the origin we started with + } + } + const item: CrawlResultItem = { url: canonicalForOutput(fetchResult.url), title: fetchResult.title, @@ -147,7 +162,7 @@ export class Crawler { // Discover links for traversal if (depth < maxDepth) { - const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser); + const newLinks = this.filterLinks(fetchResult.links, scopeOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser); // filterLinks() runs against the visited snapshot before this loop, // so two outbound links with the same canonical (e.g. /page#a and diff --git a/tests/unit/crawl/crawler.test.ts b/tests/unit/crawl/crawler.test.ts index f378bb2fe..34aff5f0d 100644 --- a/tests/unit/crawl/crawler.test.ts +++ b/tests/unit/crawl/crawler.test.ts @@ -162,6 +162,66 @@ describe('Crawler — BFS', () => { expect(urls).not.toContain('https://other.example.com/external'); }); + describe('seed redirect', () => { + // https://example.com 301s to https://www.example.com, so every link on the + // page we actually landed on carries the www host. + const redirectingFetch: FetchFn = vi.fn(async (url: string) => { + if (url === 'https://example.com') { + return makeFetchOutput('https://www.example.com', 'Home', '# Home', [ + 'https://www.example.com/a', + 'https://www.example.com/b', + 'https://other.example.com/external', + ]); + } + return makeFetchOutput(url, 'Page', '# Page', []); + }); + + it('scopes the crawl to the seed final origin, not the requested one', async () => { + const crawler = new Crawler(redirectingFetch, rawFetchFn); + const result = await crawler.crawl({ + url: 'https://example.com', + strategy: 'bfs', + max_depth: 1, + max_pages: 10, + }); + + const urls = result.pages.map((p) => p.url); + expect(urls).toContain('https://www.example.com/a'); + expect(urls).toContain('https://www.example.com/b'); + expect(result.pages.length).toBe(3); + }); + + it('still rejects off-origin links after re-anchoring', async () => { + const crawler = new Crawler(redirectingFetch, rawFetchFn); + const result = await crawler.crawl({ + url: 'https://example.com', + strategy: 'bfs', + max_depth: 1, + max_pages: 10, + }); + + expect(result.pages.map((p) => p.url)).not.toContain('https://other.example.com/external'); + }); + + it('keeps the requested origin when the seed fetch fails', async () => { + const failingSeed: FetchFn = vi.fn(async (url: string) => { + if (url === 'https://example.com') { + return { ...makeFetchOutput(url, '', '', []), error: 'fetch_failed' }; + } + return makeFetchOutput(url, 'Page', '# Page', []); + }); + const crawler = new Crawler(failingSeed, rawFetchFn); + const result = await crawler.crawl({ + url: 'https://example.com', + strategy: 'bfs', + max_depth: 1, + max_pages: 10, + }); + + expect(result.pages).toEqual([]); + }); + }); + it('does not visit the same URL twice', async () => { const crawler = new Crawler(fetchFn, rawFetchFn); await crawler.crawl({