diff --git a/CLAUDE.md b/CLAUDE.md index 4124f56..e66c733 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,11 +32,9 @@ Pipeline: `cmd/cli.ts` → `engine/stream.ts` (`XStreamer`) → `engine/browser. **Session (`XStreamer.start` in `engine/stream.ts`).** It launches the browser, attaches the interceptor to the page, and navigates to the live search URL. If the URL ends up on `/login` or `/i/flow`, the token is invalid. The run then waits for `[data-testid="cellInnerDiv"]` and enters `pollLoop`. The `finally` block closes the browser and the sink. -**Tweets come from two channels that share one listener.** -1. Passive: `XInterceptor` (`engine/interceptor.ts`) listens to every page `response` and matches `/i/api/graphql/` with `SearchTimeline`. It parses the body with `parser.ts`, which walks `data.search_by_raw_query.search_timeline.timeline.instructions[].entries[]`. It also saves the successful request as a `RequestTemplate` (URL, method, headers, body). Cookies, `sec-*` headers, and anything in `UNREPLAYABLE_HEADERS` are dropped. -2. Active: `pollLoop` replays that template with `page.evaluate(fetch(..., { credentials: "include" }))`. The page supplies the cookies and the `ct0` CSRF header, so Node never handles auth itself. The refetch's response goes through the same interceptor listener. The `refetch` result carries only status and headers, which feed rate limiting and failure counting. +**Polling.** `pollLoop` sleeps for `nextDelay()`, then calls `page.reload()` and waits for `[data-testid="cellInnerDiv"]`. Each reload makes X's own JS send a fresh `SearchTimeline` request. `XInterceptor` (`engine/interceptor.ts`) listens to every page `response` and matches `/i/api/graphql/` with `SearchTimeline`. It feeds the rate-limit headers to `RateLimitTracker` and parses the body with `parser.ts`, which walks `data.search_by_raw_query.search_timeline.timeline.instructions[].entries[]`. -**Recovery.** A refetch status other than 200 or 429 counts as a failure. After `MAX_REFETCH_FAILURES` (3) in a row, `pollLoop` clears the template and reloads the page. The reload captures a fresh template from its SearchTimeline response. A 429 is not a failure. It is handled by pacing. +**No request replay.** Replaying a captured `SearchTimeline` request with in-page `fetch` was tried and removed. X answers the replay with an empty 404 about 70% of the time on the first replay and every time on the second (likely the per-request `x-client-transaction-id`). Each rejected replay still costs rate-limit quota, so replay cost more requests per fresh page than a reload. Do not reintroduce it without a way to get a fresh token per request. **Pacing (`engine/rate-limit.ts`, `RateLimitTracker`).** It reads `x-rate-limit-{limit,remaining,reset}`. It spreads requests evenly over the remaining budget until reset, keeping 2 requests in reserve. On a 429 it blocks until the reset time. Without reset headers it backs off exponentially, from 60s up to 5 minutes. `--interval` is the floor, and `nextDelay` adds up to 10% jitter. @@ -49,4 +47,4 @@ Pipeline: `cmd/cli.ts` → `engine/stream.ts` (`XStreamer`) → `engine/browser. - The project compiles to CommonJS (there is no `"type": "module"`). `cli.ts` uses `__dirname` and `browser.ts` uses `require.resolve`. Switching to ESM means changing both. - `browser.ts` downloads Chromium on first run if the Playwright executable is missing. It runs through `playwright-extra` with the stealth plugin. - `extractCursor` in `parser.ts` is exported but unused, so cursor pagination isn't wired up. -- `README.md` lags the code. Its `--interval` minimum and default (10 and 30) differ from `schemas/stream.ts` (2 and 5), and its description of fixed-interval page reloads predates the in-page refetch. Trust the source. +- `README.md` lags the code. Its `--interval` minimum and default (10 and 30) differ from `schemas/stream.ts` (2 and 5), and its interval wording does not mention rate-limit pacing. Trust the source. diff --git a/README.md b/README.md index 0504f73..5d0b42c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Command line tool that streams X (Twitter) search results as NDJSON. -XStream opens the X search page in a headless Chromium browser, logged in with your own `auth_token` cookie. It reads tweets from the responses X sends to its own web app, reloads the page on a fixed interval, and prints each new tweet as one JSON object per line. +XStream opens the X search page in a headless Chromium browser, logged in with your own `auth_token` cookie. It reads tweets from the responses X sends to its own web app. It reloads the search page on a paced schedule that follows X's rate limit, and prints each new tweet as one JSON object per line. It does not use the official X API and needs no API key. @@ -107,21 +107,21 @@ xstream search -k "typescript" --no-headless ### Options -| Option | Description | Default | -| ------------------------ | ----------------------------------------------------------------------------------------------- | ------------- | -| `-k, --keywords ` | Search query. Supports X search operators such as `OR`, `from:`, `to:`, and `since:`. Required. | | -| `--auth-token ` | Your X `auth_token` cookie value. Required unless `AUTH_TOKEN` is set. | `AUTH_TOKEN` | -| `-l, --lang ` | Only return tweets in this language. Two letter ISO 639 code, for example `en` or `id`. | All languages | -| `--interval ` | Seconds between page reloads. Minimum 10. | `30` | -| `--max ` | Stop after this many tweets. | No limit | -| `--output ` | `stdout` or a file path. Files are appended to, not overwritten. | `stdout` | -| `--no-headless` | Show the browser window. | Headless | -| `-V, --version` | Print the version. | | -| `-h, --help` | Print help. | | +| Option | Description | Default | +| ------------------------ | ------------------------------------------------------------------------------------------------------ | ------------- | +| `-k, --keywords ` | Search query. Supports X search operators such as `OR`, `from:`, `to:`, and `since:`. Required. | | +| `--auth-token ` | Your X `auth_token` cookie value. Required unless `AUTH_TOKEN` is set. | `AUTH_TOKEN` | +| `-l, --lang ` | Only return tweets in this language. Two letter ISO 639 code, for example `en` or `id`. | All languages | +| `--interval ` | Minimum seconds between page reloads. Minimum 2. XStream waits longer when X's rate limit requires it. | `5` | +| `--max ` | Stop after this many tweets. | No limit | +| `--output ` | `stdout` or a file path. Files are appended to, not overwritten. | `stdout` | +| `--no-headless` | Show the browser window. | Headless | +| `-V, --version` | Print the version. | | +| `-h, --help` | Print help. | | ## Output -XStream writes one JSON object per line (NDJSON). Each tweet is written once per run. Within each reload, tweets are written oldest first. +XStream writes one JSON object per line (NDJSON). Each tweet is written once per run. Within each reload, tweets are written oldest first. Each reload returns the newest 20 matching tweets (see Limitations). ```json { @@ -157,7 +157,9 @@ Tweets go to stdout. Logs and status messages go to stderr. Redirecting stdout t ## Rate limits -When X returns HTTP 429, XStream doubles the reload interval, up to a maximum of 5 minutes. It keeps that interval for the rest of the run. +After each page load, XStream reads X's rate-limit headers. It spreads the remaining reloads evenly until the quota resets, and keeps 2 requests in reserve. The `--interval` value is the minimum gap between reloads. The actual gap is often longer. + +When X returns HTTP 429, XStream waits until the reset time in the response headers. If the headers are missing, it backs off exponentially, from 60 seconds up to 5 minutes. ## Troubleshooting @@ -176,8 +178,10 @@ System libraries are missing. Run `sudo npx playwright install-deps chromium`. ## Limitations 1. XStream depends on the internal format of the X web app. When X changes that format, XStream can stop working until it is updated. -2. XStream acts as your own account. Heavy use can get the account rate limited or locked. Use a longer `--interval` for long runs. -3. Scraping may violate the X Terms of Service. You are responsible for how you use this tool. +2. XStream acts as your own account. Heavy use can get the account rate limited or locked. XStream paces itself against X's rate limit. A longer `--interval` lowers the load further for long runs. +3. Each reload returns only the newest 20 matching tweets. If more than 20 matching tweets are posted between two reloads, the older ones are never written. Busy queries can lose tweets. Quiet queries do not. +4. Tweets appear with a delay of up to one poll interval after they are posted. +5. Scraping may violate the X Terms of Service. You are responsible for how you use this tool. ## Development diff --git a/src/engine/interceptor.ts b/src/engine/interceptor.ts index 8d15a65..cf36ca8 100644 --- a/src/engine/interceptor.ts +++ b/src/engine/interceptor.ts @@ -1,48 +1,11 @@ import { Tweet } from "@/types"; -import { Page, Request as PWRequest } from "playwright"; +import { Page } from "playwright"; import { createLogger } from "@/logger"; import { parseTweetsFromResponse } from "@/engine/parser"; import { RateLimitTracker } from "./rate-limit"; -export interface RequestTemplate { - url: string; - method: string; - headers: Record; - body: Buffer | null; -} - -const UNREPLAYABLE_HEADERS = new Set([ - "cookie", - "host", - "user-agent", - "referer", - "origin", - "content-length", - "connection", - "accept-encoding", -]); - -async function buildTemplate(request: PWRequest): Promise { - const rawHeaders = await request.allHeaders(); - const headers = Object.fromEntries( - Object.entries(rawHeaders).filter( - ([name]) => - !name.startsWith(":") && - !name.startsWith("sec-") && - !UNREPLAYABLE_HEADERS.has(name), - ), - ); - return { - url: request.url(), - method: request.method(), - headers, - body: request.postDataBuffer(), - }; -} - export class XInterceptor { private isAttached: boolean = false; - private template: RequestTemplate | null = null; private readonly logger = createLogger("X Interceptor"); constructor( @@ -50,14 +13,6 @@ export class XInterceptor { private readonly rateLimit: RateLimitTracker, ) {} - public getTemplate(): RequestTemplate | null { - return this.template; - } - - public invalidTemplate(): void { - this.template = null; - } - public attach(page: Page): void { if (this.isAttached) return; @@ -97,8 +52,6 @@ export class XInterceptor { return; } - this.template = await buildTemplate(response.request()); - const contentType = headers["content-type"] || ""; if (!contentType.includes("application/json")) { return; diff --git a/src/engine/stream.ts b/src/engine/stream.ts index 367e244..c0efdac 100644 --- a/src/engine/stream.ts +++ b/src/engine/stream.ts @@ -4,18 +4,12 @@ import { Browser, BrowserContext, Page } from "playwright"; import { Logger } from "tslog"; import { createLogger } from "@/logger"; import { launchBrowser } from "./browser"; -import { RequestTemplate, XInterceptor } from "./interceptor"; +import { XInterceptor } from "./interceptor"; import { RateLimitTracker } from "./rate-limit"; const MAX_SEEN_IDS = 10_000; -const MAX_REFETCH_FAILURES = 3; const JITTER_RATIO = 0.1; -interface RefetchResult { - status: number; - headers: Record; -} - function sleep(ms: number, signal: AbortSignal): Promise { return new Promise((resolve) => { const timer = setTimeout(resolve, ms); @@ -117,7 +111,6 @@ export class XStreamer { private async pollLoop(page: Page): Promise { const signal = this.abortController.signal; - let failures = 0; while (!signal.aborted) { const delay = this.nextDelay(); @@ -125,28 +118,6 @@ export class XStreamer { await sleep(delay, signal); if (signal.aborted) break; - const template = this.interceptor.getTemplate(); - if (template) { - const result = await this.refetch(page, template); - this.rateLimit.update(result.headers, result.status); - - if (result.status === 200 || result.status === 429) { - failures = 0; - continue; - } - - failures++; - this.logger.warn( - `In-page refetch failed (status ${result.status}), attempt ${failures}/${MAX_REFETCH_FAILURES}.`, - ); - if (failures < MAX_REFETCH_FAILURES) continue; - } - - this.logger.info( - "Reloading search page to capture a fresh request template.", - ); - failures = 0; - this.interceptor.invalidTemplate(); await this.reloadPage(page); } } @@ -156,40 +127,6 @@ export class XStreamer { return Math.round(base * (1 + Math.random() * JITTER_RATIO)); } - private async refetch( - page: Page, - template: RequestTemplate, - ): Promise { - const payload = { - url: template.url, - method: template.method, - headers: template.headers, - body: template.body?.toString("utf8") ?? null, - }; - - try { - return await page.evaluate(async ({ url, method, headers, body }) => { - const csrf = /(?:^|;\s*)ct0=([^;]+)/.exec(document.cookie)?.[1]; - if (csrf) headers["x-csrf-token"] = csrf; - - const response = await fetch(url, { - method, - headers, - body: body ?? undefined, - credentials: "include", - }); - await response.text(); // wait for the full body before resolving - return { - status: response.status, - headers: Object.fromEntries(response.headers.entries()), - }; - }, payload); - } catch (error) { - this.logger.warn(`In-page refetch threw: ${error}`); - return { status: 0, headers: {} }; - } - } - private async reloadPage(page: Page): Promise { await page .reload({ waitUntil: "domcontentloaded", timeout: 60000 })