From af02e3e7d92e152d31a09bc4f10f6c48c59d3373 Mon Sep 17 00:00:00 2001
From: divya0795 <12871391+divya0795@users.noreply.github.com>
Date: Sat, 1 Aug 2026 01:03:00 -0700
Subject: [PATCH] fix(crawl): re-anchor same-origin scope to the seed's
 post-redirect origin
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

crawl() derives seedOrigin from the pre-redirect input URL and never
reconciles it with where the seed actually landed. The fetch layer follows
redirects, so on any apex -> www (or http -> https) seed every link on the
fetched page carries the final host while the scope check still holds the
original — filterLinks rejects all of them as off-origin and the crawl
returns a single page.

Re-anchor once inside crawlTraversal, after the seed fetch succeeds: depth
0 is only ever the seed, so this fires exactly once and only on a page we
actually got. An unparseable final URL falls back to the requested origin.

The crawl result item already used fetchResult.url, so the output url and
the scope check now agree instead of disagreeing.
---
 src/crawl/crawler.ts             | 17 ++++++++-
 tests/unit/crawl/crawler.test.ts | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/src/crawl/crawler.ts b/src/crawl/crawler.ts
index ec6582f2f..5fdcd82b0 100644
--- a/src/crawl/crawler.ts
+++ b/src/crawl/crawler.ts
@@ -99,6 +99,12 @@ export class Crawler {
     const seenEdges = new Set<string>();
     const indexing = isIndexingEnabled();
 
+    // seedOrigin comes from the pre-redirect input URL. Re-anchored to the
+    // seed's final origin below once it has been fetched, so a seed that
+    // redirects (apex -> www, http -> https) doesn't reject every link on the
+    // page it actually landed on.
+    let scopeOrigin = seedOrigin;
+
     // Queue: [url, depth]
     const queue: Array<[string, number]> = [[input.url, 0]];
     visited.add(canonicalForCrawl(input.url));
@@ -132,6 +138,15 @@ export class Crawler {
         continue;
       }
 
+      // depth 0 is only ever the seed: scope the crawl to where it landed.
+      if (depth === 0) {
+        try {
+          scopeOrigin = new URL(fetchResult.url).origin;
+        } catch {
+          // unparseable final URL — keep the origin we started with
+        }
+      }
+
       const item: CrawlResultItem = {
         url: canonicalForOutput(fetchResult.url),
         title: fetchResult.title,
@@ -147,7 +162,7 @@ export class Crawler {
 
       // Discover links for traversal
       if (depth < maxDepth) {
-        const newLinks = this.filterLinks(fetchResult.links, seedOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
+        const newLinks = this.filterLinks(fetchResult.links, scopeOrigin, visited, input.include_patterns, input.exclude_patterns, robotsParser);
 
         // filterLinks() runs against the visited snapshot before this loop,
         // so two outbound links with the same canonical (e.g. /page#a and
diff --git a/tests/unit/crawl/crawler.test.ts b/tests/unit/crawl/crawler.test.ts
index f378bb2fe..34aff5f0d 100644
--- a/tests/unit/crawl/crawler.test.ts
+++ b/tests/unit/crawl/crawler.test.ts
@@ -162,6 +162,66 @@ describe('Crawler — BFS', () => {
     expect(urls).not.toContain('https://other.example.com/external');
   });
 
+  describe('seed redirect', () => {
+    // https://example.com 301s to https://www.example.com, so every link on the
+    // page we actually landed on carries the www host.
+    const redirectingFetch: FetchFn = vi.fn(async (url: string) => {
+      if (url === 'https://example.com') {
+        return makeFetchOutput('https://www.example.com', 'Home', '# Home', [
+          'https://www.example.com/a',
+          'https://www.example.com/b',
+          'https://other.example.com/external',
+        ]);
+      }
+      return makeFetchOutput(url, 'Page', '# Page', []);
+    });
+
+    it('scopes the crawl to the seed final origin, not the requested one', async () => {
+      const crawler = new Crawler(redirectingFetch, rawFetchFn);
+      const result = await crawler.crawl({
+        url: 'https://example.com',
+        strategy: 'bfs',
+        max_depth: 1,
+        max_pages: 10,
+      });
+
+      const urls = result.pages.map((p) => p.url);
+      expect(urls).toContain('https://www.example.com/a');
+      expect(urls).toContain('https://www.example.com/b');
+      expect(result.pages.length).toBe(3);
+    });
+
+    it('still rejects off-origin links after re-anchoring', async () => {
+      const crawler = new Crawler(redirectingFetch, rawFetchFn);
+      const result = await crawler.crawl({
+        url: 'https://example.com',
+        strategy: 'bfs',
+        max_depth: 1,
+        max_pages: 10,
+      });
+
+      expect(result.pages.map((p) => p.url)).not.toContain('https://other.example.com/external');
+    });
+
+    it('keeps the requested origin when the seed fetch fails', async () => {
+      const failingSeed: FetchFn = vi.fn(async (url: string) => {
+        if (url === 'https://example.com') {
+          return { ...makeFetchOutput(url, '', '', []), error: 'fetch_failed' };
+        }
+        return makeFetchOutput(url, 'Page', '# Page', []);
+      });
+      const crawler = new Crawler(failingSeed, rawFetchFn);
+      const result = await crawler.crawl({
+        url: 'https://example.com',
+        strategy: 'bfs',
+        max_depth: 1,
+        max_pages: 10,
+      });
+
+      expect(result.pages).toEqual([]);
+    });
+  });
+
   it('does not visit the same URL twice', async () => {
     const crawler = new Crawler(fetchFn, rawFetchFn);
     await crawler.crawl({