Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ PAPER_SEARCH_DEADLINE=8
PAPER_SEARCH_PROXY_URL=
# China deployment: Crossref is primary and OpenAlex supplements arXiv metadata.
PAPER_SEARCH_REQUIRE_PROXY=false
# Optional arXiv-only HTTPS gateway; it must mirror /api/query, /abs/<id>, and /pdf/<id>.
ARXIV_GATEWAY_URL=
# Shared gateway key. Never commit a real value.
ARXIV_GATEWAY_API_KEY=
CROSSREF_API_URL=https://api.crossref.org/works
OPENALEX_API_URL=https://api.openalex.org/works
CROSSREF_MAILTO=
Expand All @@ -52,3 +56,7 @@ OPENAI_API_KEY=
OPENAI_BASE_URL=https://api.openai.com/v1
DEEPSEEK_API_KEY=
DEEPSEEK_BASE_URL=https://api.deepseek.com

# Optional: place an enabled model first so new UI tasks select it by default.
FASTREAD_DEFAULT_PROVIDER=
FASTREAD_DEFAULT_MODEL=
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@

<h1 align="center">FastRead</h1>

## 当前主线:FastRead Web

当前入口是带账户和工作区权限的 Web 应用:服务端保存论文、版本化证据、报告、聊天与总结,独立 worker 执行可恢复的后台任务。部署、离线开户、只读源迁移、备份和回滚见 [Web 部署指南](deploy/web/README.md)。API 使用 `backend/main.py`,worker 使用 `python -m app.web.worker`,两者必须指向同一 Web 数据根。生产环境使用 HTTPS 与 Secure cookie。

后文保留早期桌面、容器和模型说明作为历史参考;正式 Web 部署以以上指南为准。

<p align="center">
<strong>从论文原文出发,把“读过”变成可回到页码核对的理解。</strong>
</p>
Expand Down Expand Up @@ -190,6 +196,8 @@ Compose 默认只监听 `127.0.0.1:3015`,不直接暴露后端端口;运行
| `FASTWRITE_ALLOWED_ORIGINS` | 显式允许的远程 FastWrite origin,逗号分隔 | 空(仅 loopback) |
| `INTEGRATION_DATA_DIR` | FastNews 缓存、专题综合和交接包目录 | `data/integrations` |
| `PAPER_SEARCH_DEADLINE` | 近邻论文冷检索总时限(秒) | `8` |
| `ARXIV_GATEWAY_URL` | 可选的 arXiv 专用 HTTPS 网关;仅镜像查询、摘要页和 PDF 路径 | 空 |
| `ARXIV_GATEWAY_API_KEY` | arXiv 网关共享访问密钥(只放本地环境,不入库) | 空 |

## 验证

Expand Down
8 changes: 8 additions & 0 deletions backend/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ PAPER_SEARCH_DEADLINE=8
PAPER_SEARCH_PROXY_URL=
# 境内部署默认直连:Crossref 主检索,OpenAlex 补充 arXiv。
PAPER_SEARCH_REQUIRE_PROXY=false
# 可选 arXiv 专用 HTTPS 网关。仅转发 /api/query、/abs/<id>、/pdf/<id>。
ARXIV_GATEWAY_URL=
# 与网关共享的访问密钥;不要提交真实值。
ARXIV_GATEWAY_API_KEY=
CROSSREF_API_URL=https://api.crossref.org/works
OPENALEX_API_URL=https://api.openalex.org/works
CROSSREF_MAILTO=
Expand All @@ -42,3 +46,7 @@ OPENAI_API_KEY=
OPENAI_BASE_URL=https://api.openai.com/v1
DEEPSEEK_API_KEY=
DEEPSEEK_BASE_URL=https://api.deepseek.com

# 可选:将已启用的指定模型稳定置于模型列表首位,作为前端新任务默认值。
FASTREAD_DEFAULT_PROVIDER=
FASTREAD_DEFAULT_MODEL=
97 changes: 97 additions & 0 deletions backend/app/services/arxiv_gateway.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Narrow transport routing for the optional FastRead arXiv gateway."""

from __future__ import annotations

from dataclasses import dataclass
import os
import re
from urllib.parse import urlsplit, urlunsplit


ARXIV_HOSTS = frozenset({"arxiv.org", "www.arxiv.org", "export.arxiv.org"})
ARXIV_GATEWAY_KEY_HEADER = "X-FastRead-Gateway-Key"
_ARXIV_DOCUMENT_PATH = re.compile(r"^/(?:abs|pdf)/[A-Za-z0-9._/-]+$")


class ArxivGatewayConfigError(ValueError):
"""The configured arXiv-only gateway is incomplete or unsafe."""


@dataclass(frozen=True)
class ArxivRequestRoute:
request_url: str
headers: dict[str, str]
via_gateway: bool


def is_arxiv_url(url: str) -> bool:
parsed = urlsplit(str(url or "").strip())
return parsed.scheme.lower() in {"http", "https"} and (
parsed.hostname or ""
).lower() in ARXIV_HOSTS


def _allowed_arxiv_path(hostname: str, path: str) -> bool:
if hostname == "export.arxiv.org" and path == "/api/query":
return True
return bool(_ARXIV_DOCUMENT_PATH.fullmatch(path))


def route_arxiv_request(
url: str,
*,
gateway_url: str | None = None,
api_key: str | None = None,
) -> ArxivRequestRoute:
"""Map only an official arXiv request onto the configured gateway.

The gateway mirrors arXiv's ``/api/query``, ``/abs/<id>`` and
``/pdf/<id>`` paths. No arbitrary destination URL is sent to it, which
keeps this feature from becoming a general-purpose proxy.
"""

original = str(url or "").strip()
parsed = urlsplit(original)
hostname = (parsed.hostname or "").lower()
if hostname not in ARXIV_HOSTS:
return ArxivRequestRoute(original, {}, False)

configured_gateway = (
os.getenv("ARXIV_GATEWAY_URL", "").strip()
if gateway_url is None
else str(gateway_url or "").strip()
)
if not configured_gateway:
return ArxivRequestRoute(original, {}, False)
if not _allowed_arxiv_path(hostname, parsed.path):
raise ArxivGatewayConfigError(
f"不支持的 arXiv 网关路径: {parsed.path or '/'}"
)

gateway = urlsplit(configured_gateway)
if gateway.scheme.lower() != "https" or not gateway.hostname:
raise ArxivGatewayConfigError("ARXIV_GATEWAY_URL 必须是有效的 HTTPS URL")
if gateway.username or gateway.password or gateway.query or gateway.fragment:
raise ArxivGatewayConfigError(
"ARXIV_GATEWAY_URL 不能包含凭据、查询参数或片段"
)
configured_key = (
os.getenv("ARXIV_GATEWAY_API_KEY", "").strip()
if api_key is None
else str(api_key or "").strip()
)
if not configured_key:
raise ArxivGatewayConfigError(
"配置 ARXIV_GATEWAY_URL 时必须同时配置 ARXIV_GATEWAY_API_KEY"
)

base_path = gateway.path.rstrip("/")
routed_path = f"{base_path}{parsed.path}"
request_url = urlunsplit(
(gateway.scheme, gateway.netloc, routed_path, parsed.query, "")
)
return ArxivRequestRoute(
request_url=request_url,
headers={ARXIV_GATEWAY_KEY_HEADER: configured_key},
via_gateway=True,
)
20 changes: 15 additions & 5 deletions backend/app/services/chat_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,11 @@ def _task_retrieval(
question: str,
history: list[dict] | None = None,
limit: int = 8,
payload: dict | None = None,
use_vectors: bool = True,
vector_query=None,
) -> tuple[dict, list[dict], dict]:
payload = _load_task(task_id)
payload = payload if payload is not None else _load_task(task_id)
paper_chunks = _paper_chunks(task_id, payload)
retrieval_query = _retrieval_query(question, history)
diagnostics = {
Expand Down Expand Up @@ -346,11 +349,11 @@ def _task_retrieval(
return payload, chunks, diagnostics

vector_chunks: list[dict] = []
if vector_index_capability()[0]:
if use_vectors and vector_index_capability()[0]:
try:
vector_chunks = [
chunk
for chunk in VectorStoreManager().query(task_id, retrieval_query, n_results=limit)
for chunk in (vector_query(retrieval_query, limit) if vector_query else VectorStoreManager().query(task_id, retrieval_query, n_results=limit))
if (chunk.get("metadata") or {}).get("source_type") == "paper_page"
and (chunk.get("distance") is None or float(chunk.get("distance")) <= 0.9)
]
Expand Down Expand Up @@ -545,6 +548,8 @@ def chat(
provider_id: str,
model_name: str,
scope: str = "task",
artifacts=None,
model_client=None,
) -> dict:
if scope == "library":
chunks = _library_chunks(question, history)
Expand All @@ -567,15 +572,20 @@ def chat(

if not task_id:
raise ValueError("当前论文问答需要 task_id")
payload, chunks, retrieval = _task_retrieval(task_id, question, history)
payload, chunks, retrieval = _task_retrieval(
task_id, question, history,
payload=artifacts.read_result(task_id) if artifacts is not None else None,
use_vectors=artifacts is None or hasattr(artifacts, "vector_query"),
vector_query=artifacts.vector_query if artifacts is not None and hasattr(artifacts, "vector_query") else None,
)
if not chunks:
status = "requested_page_missing" if retrieval.get("strategy") == "requested_page_missing" else "retrieval_miss"
return _grounding_failure(status, retrieval=retrieval)
context = _context(chunks) if chunks else "(未找到相关分页原文)"
messages = [{"role": "system", "content": PAPER_SYSTEM_PROMPT.format(context=context)}]
messages.extend(history[-20:])
messages.append({"role": "user", "content": question})
gpt = _get_gpt(provider_id, model_name)
gpt = model_client or _get_gpt(provider_id, model_name)
response = create_chat_completion(
gpt.client,
model=gpt.model,
Expand Down
Loading
Loading