diff --git a/messages/en.json b/messages/en.json index 76f8403..889b70d 100644 --- a/messages/en.json +++ b/messages/en.json @@ -773,6 +773,7 @@ "privateNetwork": "The destination points to an unauthorized private network.", "responseTooLarge": "The response exceeds the maximum allowed size.", "tooManyRedirects": "Too many redirects.", + "insecureRedirect": "Refused to follow a redirect from HTTPS to HTTP.", "httpStatus": "HTTP {status}", "connectFailed": "Connection failed: {reason}", "feedRefreshing": "This feed is already being refreshed.", diff --git a/messages/es.json b/messages/es.json index 33e0f12..3c9914f 100644 --- a/messages/es.json +++ b/messages/es.json @@ -773,6 +773,7 @@ "privateNetwork": "El destino apunta a una red privada no autorizada.", "responseTooLarge": "La respuesta supera el tamaño máximo permitido.", "tooManyRedirects": "Demasiadas redirecciones.", + "insecureRedirect": "Redirección de HTTPS a HTTP rechazada.", "httpStatus": "HTTP {status}", "connectFailed": "Fallo de conexión: {reason}", "feedRefreshing": "Este feed ya se está actualizando.", diff --git a/messages/pt-BR.json b/messages/pt-BR.json index a74c9a9..61ac5a2 100644 --- a/messages/pt-BR.json +++ b/messages/pt-BR.json @@ -773,6 +773,7 @@ "privateNetwork": "O destino aponta para uma rede privada não autorizada.", "responseTooLarge": "A resposta excede o tamanho máximo permitido.", "tooManyRedirects": "Redirecionamentos demais.", + "insecureRedirect": "Redirecionamento de HTTPS para HTTP recusado.", "httpStatus": "HTTP {status}", "connectFailed": "Falha na conexão: {reason}", "feedRefreshing": "Este feed já está sendo atualizado.", diff --git a/prisma/postgres/migrations/20261112180000_article_content_hash/migration.sql b/prisma/postgres/migrations/20261112180000_article_content_hash/migration.sql new file mode 100644 index 0000000..e70fd30 --- /dev/null +++ b/prisma/postgres/migrations/20261112180000_article_content_hash/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "article" ADD COLUMN "contentHash" TEXT; diff --git a/prisma/postgres/schema.prisma b/prisma/postgres/schema.prisma index 44c1419..61a8350 100644 --- a/prisma/postgres/schema.prisma +++ b/prisma/postgres/schema.prisma @@ -155,6 +155,8 @@ model Article { contentHtml String? snippet String? imageUrl String? + /// Impressão do título, link e HTML crus. Muda quando o feed republica o item. + contentHash String? publishedAt DateTime @default(now()) fullContentHtml String? storyId String? // artigos de feeds diferentes sobre o mesmo fato compartilham este id diff --git a/prisma/schema.base.prisma b/prisma/schema.base.prisma index 07724ad..61cc3d6 100644 --- a/prisma/schema.base.prisma +++ b/prisma/schema.base.prisma @@ -149,6 +149,8 @@ model Article { contentHtml String? snippet String? imageUrl String? + /// Impressão do título, link e HTML crus. Muda quando o feed republica o item. + contentHash String? publishedAt DateTime @default(now()) fullContentHtml String? storyId String? // artigos de feeds diferentes sobre o mesmo fato compartilham este id diff --git a/prisma/sqlite/migrations/20261112180000_article_content_hash/migration.sql b/prisma/sqlite/migrations/20261112180000_article_content_hash/migration.sql new file mode 100644 index 0000000..e70fd30 --- /dev/null +++ b/prisma/sqlite/migrations/20261112180000_article_content_hash/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "article" ADD COLUMN "contentHash" TEXT; diff --git a/prisma/sqlite/schema.prisma b/prisma/sqlite/schema.prisma index e64d019..f2b39d0 100644 --- a/prisma/sqlite/schema.prisma +++ b/prisma/sqlite/schema.prisma @@ -155,6 +155,8 @@ model Article { contentHtml String? snippet String? imageUrl String? + /// Impressão do título, link e HTML crus. Muda quando o feed republica o item. + contentHash String? publishedAt DateTime @default(now()) fullContentHtml String? storyId String? // artigos de feeds diferentes sobre o mesmo fato compartilham este id diff --git a/src/lib/feeds/discover.ts b/src/lib/feeds/discover.ts index 53cba70..2e0ecd4 100644 --- a/src/lib/feeds/discover.ts +++ b/src/lib/feeds/discover.ts @@ -2,7 +2,7 @@ import "server-only"; import { staticTranslator } from "@/i18n/static"; import { LocalizedError } from "@/lib/localized-error"; import { commonPathCandidates, extractFeedCandidates, platformFeedCandidates, youtubeFeedFromHtml } from "./discover-candidates"; -import { FetchError, safeFetch, type SafeFetchResult } from "./net"; +import { FetchError, privateFeedsAllowed, safeFetch, type SafeFetchResult } from "./net"; import { looksLikeFeed, parseFeed, type ParsedFeed } from "./parse"; export type DiscoveredFeed = { url: string; title: string; siteUrl: string | null; itemCount: number; feed: ParsedFeed }; @@ -108,7 +108,7 @@ class Discovery { private async probe(url: string): Promise { try { - const res = await safeFetch(url, { timeoutMs: PROBE_TIMEOUT_MS }); + const res = await safeFetch(url, { timeoutMs: PROBE_TIMEOUT_MS, allowPrivateNetwork: privateFeedsAllowed() }); this.tried.add(res.url); if (looksLikeFeed(res.body)) return { feed: await toDiscovered(res) }; return { html: res.body, url: res.url }; @@ -166,7 +166,7 @@ export async function discoverFeeds(input: string): Promise { let pageError: unknown = null; discovery.markTried(url); try { - page = await safeFetch(url); + page = await safeFetch(url, { allowPrivateNetwork: privateFeedsAllowed() }); } catch (error) { if (isFatal(error)) throw error; pageError = error; diff --git a/src/lib/feeds/extract.ts b/src/lib/feeds/extract.ts index 72816c3..439d7f1 100644 --- a/src/lib/feeds/extract.ts +++ b/src/lib/feeds/extract.ts @@ -18,8 +18,8 @@ export type ExtractedPage = { }; /** Baixa a página e extrai o conteúdo principal (modo leitura) e os metadados. */ -export async function extractPage(url: string): Promise { - const res = await safeFetch(url, { accept: "text/html,application/xhtml+xml" }); +export async function extractPage(url: string, opts: { allowPrivateNetwork?: boolean } = {}): Promise { + const res = await safeFetch(url, { accept: "text/html,application/xhtml+xml", allowPrivateNetwork: opts.allowPrivateNetwork ?? false }); const { document } = parseHTML(res.body); const pageTitle = document.querySelector("title")?.textContent?.trim() || null; const base = document.createElement("base"); diff --git a/src/lib/feeds/net.ts b/src/lib/feeds/net.ts index e25869f..1e71489 100644 --- a/src/lib/feeds/net.ts +++ b/src/lib/feeds/net.ts @@ -1,4 +1,5 @@ import "server-only"; +import { isIP } from "node:net"; import { pinnedFetch } from "@/lib/network"; import { LocalizedError, type ServerErrorKey } from "@/lib/localized-error"; @@ -34,10 +35,38 @@ export function parseRetryAfter(value: string | null, now = Date.now()): number return Number.isNaN(date) ? undefined : Math.max(0, Math.round((date - now) / 1000)); } +/** Feeds na rede local (RSS-Bridge). Não vale para capa nem extração de artigo. */ +export const privateFeedsAllowed = () => process.env.ALLOW_PRIVATE_FEEDS === "true"; + +const MULTI_PART_SUFFIX = new Set([ + "com.br", "net.br", "org.br", "gov.br", + "co.uk", "org.uk", "com.au", "co.jp", "co.nz", "com.mx", "com.ar", "com.tr", "co.za", + "com.cn", "com.tw", "co.kr", "com.sg", "com.pt", "com.co", +]); + +function registrableDomain(hostname: string): string { + const host = hostname.replace(/\.$/, "").toLowerCase(); + if (isIP(host)) return host; + const parts = host.split(".").filter(Boolean); + if (parts.length < 2) return host; + const tail = parts.slice(-2).join("."); + if (MULTI_PART_SUFFIX.has(tail) && parts.length >= 3) return parts.slice(-3).join("."); + return tail; +} + +/** Mesmo site (domínio registrável), para aceitar uma URL nova de feed depois de um 301/308. */ +export function sameSite(a: string, b: string): boolean { + try { + return registrableDomain(new URL(a).hostname) === registrableDomain(new URL(b).hostname); + } catch { + return false; + } +} + /** fetch com timeout, limite de tamanho, redirects checados e proteção SSRF. */ export async function safeFetch( input: string, - opts: { headers?: Record; timeoutMs?: number; maxBytes?: number; accept?: string } = {}, + opts: { headers?: Record; timeoutMs?: number; maxBytes?: number; accept?: string; allowPrivateNetwork?: boolean } = {}, ): Promise { const { timeoutMs = 15000, maxBytes = 5 * 1024 * 1024 } = opts; let url = new URL(input); @@ -55,10 +84,11 @@ export async function safeFetch( headers: { "user-agent": USER_AGENT, accept: opts.accept ?? "application/rss+xml, application/atom+xml, application/xml, text/xml, text/html;q=0.8, */*;q=0.5", - ...opts.headers, + // Cache condicional só no primeiro hop: um 304 do destino do redirect não é "o feed não mudou". + ...(hop === 0 ? opts.headers : {}), }, }, - process.env.ALLOW_PRIVATE_FEEDS === "true", + opts.allowPrivateNetwork ?? false, ); } catch (error) { if (error instanceof LocalizedError) throw error; @@ -66,14 +96,17 @@ export async function safeFetch( } try { const res = connection.response; + const movedPermanently = redirects > 0 && allPermanent; + // 304 não é redirect, mesmo com Location. + if (res.status === 304) return { status: 304, url: url.toString(), headers: res.headers, body: "", movedPermanently }; if (res.status >= 300 && res.status < 400 && res.headers.get("location")) { - url = new URL(res.headers.get("location")!, url); + const next = new URL(res.headers.get("location")!, url); + if (url.protocol === "https:" && next.protocol !== "https:") throw new FetchError("insecureRedirect"); + url = next; redirects++; allPermanent &&= res.status === 301 || res.status === 308; continue; } - const movedPermanently = redirects > 0 && allPermanent; - if (res.status === 304) return { status: 304, url: url.toString(), headers: res.headers, body: "", movedPermanently }; if (!res.ok) throw new FetchError("httpStatus", { status: res.status }, res.status, parseRetryAfter(res.headers.get("retry-after"))); const reader = res.body?.getReader(); @@ -99,14 +132,25 @@ export async function safeFetch( throw new FetchError("tooManyRedirects"); } -function decode(buffer: Buffer, headers: { get(name: string): string | null }): string { - const fromHeader = /charset=([^;]+)/i.exec(headers.get("content-type") ?? "")?.[1]; +function unquoteCharset(value: string | undefined): string | undefined { + if (!value) return undefined; + const cleaned = value.trim().replace(/^["']+|["']+$/g, "").trim().toLowerCase(); + return cleaned || undefined; +} + +/** Decodifica o corpo usando o charset do header (sem aspas) ou da declaração do XML. */ +export function decodeFeedBody(buffer: Buffer, contentType: string | null): string { + const fromHeader = unquoteCharset(/charset=\s*("[^"]*"|'[^']*'|[^;\s]+)/i.exec(contentType ?? "")?.[1]); const head = buffer.subarray(0, 1024).toString("latin1"); - const fromBody = /encoding=["']([^"']+)["']/i.exec(head)?.[1] ?? /]+charset=["']?([\w-]+)/i.exec(head)?.[1]; - const charset = (fromHeader ?? fromBody ?? "utf-8").trim().toLowerCase(); + const fromBody = unquoteCharset(/encoding=["']([^"']+)["']/i.exec(head)?.[1] ?? /]+charset=["']?([\w-]+)/i.exec(head)?.[1]); + const charset = fromHeader ?? fromBody ?? "utf-8"; try { return new TextDecoder(charset).decode(buffer); } catch { return new TextDecoder("utf-8").decode(buffer); } } + +function decode(buffer: Buffer, headers: { get(name: string): string | null }): string { + return decodeFeedBody(buffer, headers.get("content-type")); +} diff --git a/src/lib/feeds/parse.ts b/src/lib/feeds/parse.ts index 6b23500..321c959 100644 --- a/src/lib/feeds/parse.ts +++ b/src/lib/feeds/parse.ts @@ -55,6 +55,8 @@ export type ParsedArticle = { publishedAt: Date; /** false: o XML não tinha data; `publishedAt` é a hora da coleta. */ hasPublishDate: boolean; + /** Impressão do título, link e HTML crus. Ausente em itens montados fora do parse. */ + contentHash?: string | null; }; /** Item que já está no banco: só guid e data (para corrigir uma data de fallback). */ @@ -67,71 +69,123 @@ export type ParsedFeed = { iconUrl: string | null; /** Itens novos, completos (HTML saneado, imagem, resumo). */ items: ParsedArticle[]; + /** Itens já gravados cujo conteúdo mudou. */ + changed: ParsedArticle[]; + /** Itens já gravados sem impressão: gravar o hash, sem reescrever o HTML. */ + hashBackfill: { guid: string; contentHash: string }[]; /** Itens já gravados (quando `existing` foi informado): não passam pelo saneamento. */ known: KnownItem[]; }; +/** O que já está no banco para um guid. */ +export type StoredHead = { publishedAt: Date; contentHash: string | null }; + type ParseOptions = { /** - * Retorna, dos guids informados, os que já existem no banco e a data gravada. Itens conhecidos - * pulam o saneamento do HTML, que é a parte cara do parse e se repetia a cada atualização. + * Retorna, dos guids informados, os que já existem no banco, a data gravada e a impressão do + * conteúdo. Itens conhecidos e inalterados pulam o saneamento do HTML. */ - existing?: (guids: string[]) => Promise>; + existing?: (guids: string[]) => Promise>; /** Itens publicados antes disto (fora da retenção) são descartados sem passar pelo saneamento. */ skipBefore?: Date; }; +const MAX_PARSED_ITEMS = 200; + export function looksLikeFeed(body: string): boolean { - const head = body.slice(0, 2000).trimStart(); + const head = body.slice(0, 4000).trimStart(); let i = 0; const skipWs = () => { while (i < head.length && /\s/.test(head[i])) i++; }; + const startsWith = (token: string) => head.slice(i, i + token.length).toLowerCase() === token; - if (head.startsWith("", i + 5); - if (endDecl === -1) return false; - i = endDecl + 2; + // Declaração XML, stylesheet, comentário e DOCTYPE vêm antes do elemento raiz. + for (;;) { skipWs(); + if (startsWith("", i + 2); + if (end === -1) return false; + i = end + 2; + continue; + } + if (startsWith("", i + 4); + if (end === -1) return false; + i = end + 3; + continue; + } + if (startsWith("", i + 4); - if (endComment === -1) return false; - i = endComment + 3; - skipWs(); - } + return /^<(rss|feed|rdf:RDF)(\W|$)/i.test(head.slice(i)); +} - const rest = head.slice(i); - const m = /^<(rss|feed|rdf:RDF)(\W|$)/i.exec(rest); - return m !== null; +/** Fim do DOCTYPE (depois do `>`), ou -1. Respeita aspas e o subconjunto interno `[...]`. */ +function skipDoctype(head: string, start: number): number { + let quote: string | null = null; + let bracket = 0; + for (let j = start + 9; j < head.length; j++) { + const ch = head[j]; + if (quote) { + if (ch === quote) quote = null; + continue; + } + if (ch === '"' || ch === "'") { + quote = ch; + continue; + } + if (ch === "[") bracket++; + else if (ch === "]" && bracket > 0) bracket--; + else if (ch === ">" && bracket === 0) return j + 1; + } + return -1; } export async function parseFeed(xml: string, feedUrl: string, opts: ParseOptions = {}): Promise { const feed = await parser.parseString(xml); const siteUrl = feed.link ? safeUrl(feed.link, feedUrl) : null; - // Fase 1 (barata): link, data e guid de cada item. - const all = feed.items.slice(0, 200).map((item) => { + // Fase 1 (barata): link, data e guid de cada item. A retenção filtra antes do teto, e o teto + // fica com os mais novos — um feed antigo-primeiro não esconde o item 201. + const all = feed.items.map((item) => { const rawLink = item.link ? safeUrl(item.link, siteUrl ?? feedUrl) : null; const link = rawLink ? (canonicalizeUrl(rawLink) ?? rawLink) : null; const rawHtml = item["content:encoded"] ?? item.content ?? item.summary ?? ""; const { publishedAt, hasPublishDate } = resolvePublishedAt(item, link); const dateKey = hasPublishDate ? publishedAt.toISOString() : ""; + const title = item.title ?? ""; // Sem guid/id/link: o conteúdo entra no hash para itens sem título nem data não colidirem entre si. - const guid = itemGuid(item.guid ?? item.id, link, `${item.title ?? ""}|${dateKey}|${rawHtml || item.contentSnippet || ""}`); - return { item, link, rawHtml, publishedAt, hasPublishDate, guid }; + const guid = itemGuid(item.guid ?? item.id, link, `${title}|${dateKey}|${rawHtml || item.contentSnippet || ""}`); + const contentHash = createHash("sha1").update(`${title}\0${link ?? ""}\0${rawHtml}`).digest("hex"); + return { item, link, rawHtml, publishedAt, hasPublishDate, guid, contentHash }; }); - const heads = opts.skipBefore ? all.filter((h) => h.publishedAt >= opts.skipBefore!) : all; + const inWindow = opts.skipBefore ? all.filter((h) => h.publishedAt >= opts.skipBefore!) : all; + const heads = capNewest(inWindow, MAX_PARSED_ITEMS); const stored = opts.existing && heads.length ? await opts.existing([...new Set(heads.map((h) => h.guid))]) : null; const known: KnownItem[] = []; const items: ParsedArticle[] = []; + const changed: ParsedArticle[] = []; + const hashBackfill: { guid: string; contentHash: string }[] = []; - // Fase 2 (cara): saneamento e extração só dos itens novos. - for (const { item, link, rawHtml, publishedAt, hasPublishDate, guid } of heads) { - if (stored?.has(guid)) { + // Fase 2 (cara): saneamento só dos itens novos ou cujo conteúdo mudou. + for (const head of heads) { + const { item, link, rawHtml, publishedAt, hasPublishDate, guid, contentHash } = head; + const prev = stored?.get(guid); + if (prev && prev.contentHash === contentHash) { + known.push({ guid, publishedAt, hasPublishDate }); + continue; + } + if (prev && prev.contentHash == null) { + hashBackfill.push({ guid, contentHash }); known.push({ guid, publishedAt, hasPublishDate }); continue; } @@ -144,7 +198,7 @@ export async function parseFeed(xml: string, feedUrl: string, opts: ParseOptions (item.enclosure?.type?.startsWith("image/") ? item.enclosure.url : null) ?? firstImage(contentHtml); - items.push({ + const article: ParsedArticle = { guid, url: link, title: (item.title && stripHtml(item.title)) || truncate(text, 80) || "(sem título)", @@ -154,7 +208,10 @@ export async function parseFeed(xml: string, feedUrl: string, opts: ParseOptions imageUrl: imageUrl ?? null, publishedAt, hasPublishDate, - }); + contentHash, + }; + if (prev) changed.push(article); + else items.push(article); } return { @@ -163,10 +220,24 @@ export async function parseFeed(xml: string, feedUrl: string, opts: ParseOptions description: feed.description ?? null, iconUrl: (feed.image as { url?: string } | undefined)?.url ?? (siteUrl ? faviconFor(siteUrl) : null), items, + changed, + hashBackfill, known, }; } +/** Mantém a ordem do documento, mas no máximo os `limit` mais recentes. */ +function capNewest(items: T[], limit: number): T[] { + if (items.length <= limit) return items; + const keep = new Set( + [...items] + .sort((a, b) => +b.publishedAt - +a.publishedAt || (a.guid < b.guid ? -1 : a.guid > b.guid ? 1 : 0)) + .slice(0, limit) + .map((item) => item.guid), + ); + return items.filter((item) => keep.has(item.guid)); +} + export function faviconFor(siteUrl: string): string | null { try { return new URL("/favicon.ico", siteUrl).toString(); diff --git a/src/lib/feeds/refresh.ts b/src/lib/feeds/refresh.ts index 1f63bbe..46ed235 100644 --- a/src/lib/feeds/refresh.ts +++ b/src/lib/feeds/refresh.ts @@ -4,7 +4,7 @@ import { setArticleStates } from "@/lib/article-state"; import { getAppSettings } from "@/lib/app-settings"; import { withLock } from "@/lib/jobs/lock"; import { LocalizedError, serializeError } from "@/lib/localized-error"; -import { FetchError, safeFetch } from "./net"; +import { FetchError, privateFeedsAllowed, safeFetch, sameSite } from "./net"; import { parseFeed, type ParsedFeed } from "./parse"; import { assignStories } from "./stories"; import { INGEST_UNREAD_MAX_AGE_MS, SUBSCRIBE_UNREAD_MAX_AGE_MS } from "./unread-age"; @@ -71,22 +71,48 @@ export async function ingestItems(feedId: string, items: ParsedFeed["items"], re /** Guids deste feed já gravados, com a data de publicação gravada. */ async function storedGuids(feedId: string, guids: string[]) { - const rows = await db.article.findMany({ where: { feedId, guid: { in: guids } }, select: { guid: true, publishedAt: true } }); - return new Map(rows.map((r) => [r.guid, r.publishedAt])); + const rows = await db.article.findMany({ + where: { feedId, guid: { in: guids } }, + select: { guid: true, publishedAt: true, contentHash: true }, + }); + return new Map(rows.map((r) => [r.guid, { publishedAt: r.publishedAt, contentHash: r.contentHash }])); } /** * Itens já gravados: só corrige a data quando o XML traz uma real mais antiga que a gravada * (fallback "agora" ou bumpado); nunca adianta. Evita um UPDATE por item a cada refresh. */ -async function fixKnownDates(feedId: string, known: ParsedFeed["known"], stored: Map) { +async function fixKnownDates(feedId: string, known: ParsedFeed["known"], stored: Map) { for (const item of known) { const current = stored.get(item.guid); - if (!item.hasPublishDate || !current || current <= item.publishedAt) continue; + if (!item.hasPublishDate || !current || current.publishedAt <= item.publishedAt) continue; await db.article.updateMany({ where: { feedId, guid: item.guid }, data: { publishedAt: item.publishedAt } }); } } +/** Grava a impressão atual dos itens antigos que ainda não tinham uma, sem reescrever o HTML. */ +async function backfillContentHashes(feedId: string, rows: ParsedFeed["hashBackfill"]) { + for (const row of rows) { + await db.article.updateMany({ + where: { feedId, guid: row.guid, contentHash: null }, + data: { contentHash: row.contentHash }, + }); + } +} + +/** Título, HTML, resumo e imagem de um item já gravado, quando o feed republicou o conteúdo. */ +async function updateChangedItems(feedId: string, items: ParsedFeed["changed"]) { + for (const item of items) { + const { hasPublishDate, guid, publishedAt, ...data } = item; + await db.article.updateMany({ where: { feedId, guid }, data }); + if (!hasPublishDate) continue; + await db.article.updateMany({ + where: { feedId, guid, publishedAt: { gt: publishedAt } }, + data: { publishedAt }, + }); + } +} + /** Marca como lidos, para todos os assinantes, artigos deste feed publicados há mais de 14 dias. */ export async function markStaleUnread(feedId: string) { const cutoff = new Date(Date.now() - INGEST_UNREAD_MAX_AGE_MS); @@ -133,6 +159,7 @@ async function refreshFeedUnlocked(feedId: string): Promise { try { const res = await safeFetch(feed.url, { + allowPrivateNetwork: privateFeedsAllowed(), headers: { ...(feed.etag ? { "if-none-match": feed.etag } : {}), ...(feed.lastModified ? { "if-modified-since": feed.lastModified } : {}), @@ -142,7 +169,8 @@ async function refreshFeedUnlocked(feedId: string): Promise { const next = new Date(Date.now() + nextFetchDelayMinutes(refreshIntervalMinutes, res.headers.get("cache-control")) * 60_000); // 301/308: passa a buscar no endereço novo. Se outro feed já usa essa URL (ou outro refresh a // tomou ao mesmo tempo), o conflito de unicidade só mantém a URL antiga: não é falha do feed. - if (res.movedPermanently && res.url !== feed.url) { + // Só troca o endereço gravado quando o destino permanente continua no mesmo site. + if (res.movedPermanently && res.url !== feed.url && sameSite(feed.url, res.url)) { await db.feed.update({ where: { id: feedId }, data: { url: res.url } }).catch((error) => { if (!isUniqueConflict(error)) throw error; }); @@ -156,13 +184,15 @@ async function refreshFeedUnlocked(feedId: string): Promise { return { newArticleIds: [] }; } - let stored = new Map(); + let stored = new Map(); const cutoff = await retentionCutoff(); const parsed = await parseFeed(res.body, res.url, { existing: async (guids) => (stored = await storedGuids(feedId, guids)), skipBefore: cutoff, }); await fixKnownDates(feedId, parsed.known, stored); + await backfillContentHashes(feedId, parsed.hashBackfill); + await updateChangedItems(feedId, parsed.changed); const createdIds = await ingestItems(feedId, parsed.items, cutoff); await markStaleUnread(feedId); const newArticleIds = await freshCreatedIds(createdIds); @@ -239,15 +269,12 @@ export async function subscribe( let feed = await db.feed.findUnique({ where: { url } }); let parsed = opts.parsed; if (!feed && !parsed) { - const res = await safeFetch(url); - // Endereço antigo de um feed que mudou (301/308): usa o feed já cadastrado na URL nova em vez - // de criar um duplicado com os mesmos artigos. - if (res.movedPermanently) { - const moved = await db.feed.findUnique({ where: { url: res.url } }); - if (moved) { - url = res.url; - feed = moved; - } + const res = await safeFetch(url, { allowPrivateNetwork: privateFeedsAllowed() }); + // Endereço antigo de um feed que mudou (301/308) no mesmo site: grava a URL final, e reaproveita + // a linha que já estiver lá, em vez de duplicar os artigos. + if (res.movedPermanently && sameSite(url, res.url)) { + url = res.url; + feed = await db.feed.findUnique({ where: { url } }); } if (!feed) parsed = await parseFeed(res.body, res.url); } diff --git a/src/lib/feeds/saved-links.ts b/src/lib/feeds/saved-links.ts index 776d739..258a8db 100644 --- a/src/lib/feeds/saved-links.ts +++ b/src/lib/feeds/saved-links.ts @@ -2,6 +2,7 @@ import "server-only"; import { db } from "@/lib/db"; import { accessibleArticles } from "@/lib/article-state"; import { stripHtml, truncate } from "@/lib/utils"; +import { privateFeedsAllowed } from "./net"; import { extractPage } from "./extract"; // Links avulsos salvos para ler depois ficam num feed virtual por usuário. Ele não tem @@ -68,7 +69,7 @@ export async function saveLink(userId: string, rawUrl: string, feedTitle: string return { articleId: previous.id, existed: true }; } - const page = await extractPage(url); + const page = await extractPage(url, { allowPrivateNetwork: privateFeedsAllowed() }); const host = new URL(page.url).hostname.replace(/^www\./, ""); const text = page.excerpt ?? (page.html ? truncate(stripHtml(page.html), 280) : null); const article = await db.article.create({ diff --git a/src/lib/localized-error.ts b/src/lib/localized-error.ts index 215ff23..a8f2b71 100644 --- a/src/lib/localized-error.ts +++ b/src/lib/localized-error.ts @@ -7,6 +7,7 @@ export type ServerErrorKey = | "privateNetwork" | "responseTooLarge" | "tooManyRedirects" + | "insecureRedirect" | "httpStatus" | "connectFailed" | "feedRefreshing" diff --git a/tests/db-integration.test.ts b/tests/db-integration.test.ts index 049d175..aa4ca9f 100644 --- a/tests/db-integration.test.ts +++ b/tests/db-integration.test.ts @@ -433,6 +433,7 @@ describe("subscribe backlog", () => { imageUrl: null, publishedAt, hasPublishDate, + contentHash: null, }); it("marks historical and undated items read, keeps yesterday unread", async () => { @@ -443,6 +444,8 @@ describe("subscribe backlog", () => { description: null, iconUrl: null, known: [], + changed: [], + hashBackfill: [], items: [ parsedItem("old", new Date(Date.now() - 14 * 86400000)), parsedItem("fresh", new Date(Date.now() - 3600_000)), diff --git a/tests/feeds.test.ts b/tests/feeds.test.ts index 2867242..47b1f33 100644 --- a/tests/feeds.test.ts +++ b/tests/feeds.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { looksLikeFeed, parseFeed } from "@/lib/feeds/parse"; +import { decodeFeedBody, sameSite } from "@/lib/feeds/net"; import { buildOpml, parseOpml } from "@/lib/feeds/opml"; import { parseSrcset, sanitizeArticleHtml } from "@/lib/feeds/sanitize"; @@ -32,6 +33,12 @@ describe("parseFeed", () => { expect(looksLikeFeed(rss)).toBe(true); expect(looksLikeFeed(atom)).toBe(true); expect(looksLikeFeed("")).toBe(false); + expect( + looksLikeFeed(` + +`), + ).toBe(true); + expect(looksLikeFeed(`\n\n`)).toBe(true); }); it("normaliza itens RSS com URLs absolutas e HTML sanitizado", async () => { @@ -56,7 +63,7 @@ describe("parseFeed", () => { const feed = await parseFeed(rss, "https://exemplo.com/feed.xml", { existing: async (guids) => { asked.push(guids); - return new Map([["post-1", new Date("2026-09-15T00:00:00Z")]]); + return new Map([["post-1", { publishedAt: new Date("2026-09-15T00:00:00Z"), contentHash: null }]]); }, }); expect(asked).toEqual([["post-1", "https://exemplo.com/posts/2"]]); @@ -111,6 +118,38 @@ describe("parseFeed", () => { const feed = await parseFeed(xml, "https://nodate.test/feed"); expect(feed.items[0].hasPublishDate).toBe(false); }); + + it("filtra pela retenção antes do teto de 200 itens", async () => { + const old = Array.from({ length: 200 }, (_, i) => `old ${i}old-${i}Mon, 01 Jan 2024 00:00:00 GMT`).join(""); + const xml = `T${old} + novonew-1Mon, 14 Sep 2026 10:00:00 GMT + `; + const feed = await parseFeed(xml, "https://exemplo.com/feed.xml", { skipBefore: new Date("2026-01-01T00:00:00Z") }); + expect(feed.items.map((item) => item.guid)).toEqual(["new-1"]); + }); + + it("marca item republicado para atualização", async () => { + const first = await parseFeed(rss, "https://exemplo.com/feed.xml"); + const feed = await parseFeed(rss.replace("Olá", "Olá de novo"), "https://exemplo.com/feed.xml", { + existing: async () => new Map([["post-1", { publishedAt: first.items[0].publishedAt, contentHash: first.items[0].contentHash ?? null }]]), + }); + expect(feed.items.map((item) => item.guid)).toEqual(["https://exemplo.com/posts/2"]); + expect(feed.changed).toHaveLength(1); + expect(feed.changed[0].guid).toBe("post-1"); + expect(feed.changed[0].contentHtml).toContain("Olá de novo"); + }); +}); + +describe("decode and same site", () => { + it("tira aspas do charset antes de decodificar", () => { + expect(decodeFeedBody(Buffer.from("café", "latin1"), 'text/xml; charset="iso-8859-1"')).toBe("café"); + }); + + it("aceita redirect permanente só no mesmo site", () => { + expect(sameSite("https://www.example.com/rss", "https://feeds.example.com/rss.xml")).toBe(true); + expect(sameSite("https://news.com.br/rss", "https://www.news.com.br/atom")).toBe(true); + expect(sameSite("https://example.com/rss", "https://evil.test/rss")).toBe(false); + }); }); describe("sanitizeArticleHtml", () => {