diff --git a/README.md b/README.md index 66c61ad..0a535cb 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,12 @@ The `fable-engine` server exposes the unified `fable_session` tool adhering to J | `unlock_execution` | Gatekeeper | `session_name`, `rationale` | Evaluates time-lock clock and refinement criteria to unlock workspace modifications. | | `check_auto_update` | Sync | `session_name?` | Probes upstream repository for updates without blocking execution. | | `apply_auto_update` | Sync | `preserve_cortex?` | Updates skills and configuration while preserving local cortical learning. | +| `scrape_web` | Research | `target`, `auto_log_epistemic?` | Scrapes web page URL or performs free DuckDuckGo web search returning Markdown. | +| `scrape_youtube` | Research | `target`, `auto_log_epistemic?` | Scrapes YouTube video metadata and public caption/transcript tracks returning Markdown. | +| `scrape_reddit` | Research | `target`, `auto_log_epistemic?` | Scrapes Reddit threads, comments, or subreddits via free JSON API returning Markdown. | +| `scrape_x` | Research | `target`, `auto_log_epistemic?` | Scrapes individual Tweet URLs via Twitter syndication API and best-effort handle lookups returning Markdown. | +| `scrape_github` | Research | `target`, `auto_log_epistemic?` | Scrapes GitHub repository metadata, READMEs, or repository search returning Markdown. | +| `scrape_arxiv` | Research | `target`, `auto_log_epistemic?` | Scrapes arXiv academic papers, abstracts, and PDF links returning Markdown. | --- diff --git a/fable_engine/actions/__init__.py b/fable_engine/actions/__init__.py index 0372935..8c2035c 100644 --- a/fable_engine/actions/__init__.py +++ b/fable_engine/actions/__init__.py @@ -70,6 +70,14 @@ _handle_validate_preflight_design, _handle_list_design_archetypes, ) +from fable_engine.actions.scrapers import ( + _handle_scrape_web, + _handle_scrape_youtube, + _handle_scrape_reddit, + _handle_scrape_x, + _handle_scrape_github, + _handle_scrape_arxiv, +) ACTION_DISPATCH: Dict[str, Callable[[Dict[str, Any]], str]] = { "create_session": _handle_create_session, @@ -236,6 +244,19 @@ "design_archetypes": _handle_list_design_archetypes, "list_archetypes": _handle_list_design_archetypes, "archetypes": _handle_list_design_archetypes, + "scrape_web": _handle_scrape_web, + "web_scrape": _handle_scrape_web, + "scrape_youtube": _handle_scrape_youtube, + "youtube_scrape": _handle_scrape_youtube, + "scrape_reddit": _handle_scrape_reddit, + "reddit_scrape": _handle_scrape_reddit, + "scrape_x": _handle_scrape_x, + "x_scrape": _handle_scrape_x, + "twitter_scrape": _handle_scrape_x, + "scrape_github": _handle_scrape_github, + "github_scrape": _handle_scrape_github, + "scrape_arxiv": _handle_scrape_arxiv, + "arxiv_scrape": _handle_scrape_arxiv, } diff --git a/fable_engine/actions/scrapers.py b/fable_engine/actions/scrapers.py new file mode 100644 index 0000000..4c0baaf --- /dev/null +++ b/fable_engine/actions/scrapers.py @@ -0,0 +1,116 @@ +""" +Fable Engine Research Scraping Action Handlers. +Exposes zero-cost scrapers for Web, YouTube, Reddit, X, GitHub, and arXiv as individual MCP actions. +Integrates retrieved evidence into Fable Session epistemic ledger as [HYPOTHESIS] candidate items. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict + +from fable_engine.scrapers import ( + ResearchResult, + scrape_arxiv, + scrape_github, + scrape_reddit, + scrape_web, + scrape_x, + scrape_youtube, +) +from fable_engine.session import ACTIVE_SESSIONS, get_or_load_session + +logger = logging.getLogger("fable-engine.actions.scrapers") + + +def _get_target_from_args(args: Dict[str, Any]) -> str: + """Extracts target string from args using common parameter keys.""" + target = args.get("target") or args.get("query") or args.get("url") or args.get("target_resource") or args.get("claim") or "" + return str(target).strip() + + +def _auto_log_epistemic_if_requested(args: Dict[str, Any], action_name: str, target: str, res: ResearchResult): + """ + Optionally logs successful scraped result as a [HYPOTHESIS] epistemic item if requested. + Failed retrievals (res.ok == False) are strictly excluded from epistemic logging. + Scraped raw web data is tagged as [HYPOTHESIS] so agents are required to + cross-check and verify evidence before promoting claims to [PROVEN]. + """ + auto_log = args.get("auto_log_epistemic") or args.get("auto_log") + if not auto_log: + return + + # Do not auto-log failed retrievals + if not res.ok: + logger.debug(f"Skipping auto_log_epistemic for {action_name}: retrieval failed.") + return + + session_name = args.get("session_name", "").strip() + if not session_name: + if ACTIVE_SESSIONS: + session_name = next(iter(ACTIVE_SESSIONS.keys())) + else: + return + try: + session = get_or_load_session(session_name) + claim = f"Retrieved {action_name} research candidate: '{target}' ({res.title or 'Untitled'})" + evidence_content = res.to_markdown() + evidence_snippet = evidence_content[:1500] if len(evidence_content) > 1500 else evidence_content + # Log as HYPOTHESIS candidate evidence + session.log_epistemic_item("HYPOTHESIS", claim, evidence_snippet) + except Exception as e: + logger.warning(f"Could not auto-log epistemic hypothesis for {action_name}: {e}") + + +def _handle_scrape_web(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'url' or 'query' for scrape_web." + res = scrape_web(target) + _auto_log_epistemic_if_requested(args, "scrape_web", target, res) + return res.to_markdown() + + +def _handle_scrape_youtube(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'url' or 'video_id' for scrape_youtube." + res = scrape_youtube(target) + _auto_log_epistemic_if_requested(args, "scrape_youtube", target, res) + return res.to_markdown() + + +def _handle_scrape_reddit(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'url' or 'subreddit' for scrape_reddit." + res = scrape_reddit(target) + _auto_log_epistemic_if_requested(args, "scrape_reddit", target, res) + return res.to_markdown() + + +def _handle_scrape_x(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'url' or 'handle' for scrape_x." + res = scrape_x(target) + _auto_log_epistemic_if_requested(args, "scrape_x", target, res) + return res.to_markdown() + + +def _handle_scrape_github(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'repo' or 'query' for scrape_github." + res = scrape_github(target) + _auto_log_epistemic_if_requested(args, "scrape_github", target, res) + return res.to_markdown() + + +def _handle_scrape_arxiv(args: Dict[str, Any]) -> str: + target = _get_target_from_args(args) + if not target: + return "Error: Missing required parameter 'target' or 'paper_id' or 'query' for scrape_arxiv." + res = scrape_arxiv(target) + _auto_log_epistemic_if_requested(args, "scrape_arxiv", target, res) + return res.to_markdown() diff --git a/fable_engine/schema.py b/fable_engine/schema.py index 732c9db..1c5b31d 100644 --- a/fable_engine/schema.py +++ b/fable_engine/schema.py @@ -74,7 +74,13 @@ "generate_design_tokens", "generate_awwwards_scaffold", "validate_preflight_design", - "list_design_archetypes" + "list_design_archetypes", + "scrape_web", + "scrape_youtube", + "scrape_reddit", + "scrape_x", + "scrape_github", + "scrape_arxiv" ], "description": "The Fable session action to perform." }, @@ -525,6 +531,14 @@ "user_prompt": { "type": "string", "description": "Alias for prompt: high-level design prompt describing desired web experience." + }, + "target": { + "type": "string", + "description": "Target URL, ID, subreddit, repo, paper ID, handle, or search query string for research scraping actions." + }, + "auto_log_epistemic": { + "type": "boolean", + "description": "Whether to automatically record retrieved research findings as a [HYPOTHESIS] candidate item in Fable Session's epistemic ledger for subsequent cross-verification." } }, "required": ["action"] diff --git a/fable_engine/scrapers/__init__.py b/fable_engine/scrapers/__init__.py new file mode 100644 index 0000000..77c34ba --- /dev/null +++ b/fable_engine/scrapers/__init__.py @@ -0,0 +1,32 @@ +""" +Fable Engine Research Scrapers Package. +Exporting zero-cost, standard-library based scrapers for Web, YouTube, Reddit, X, GitHub, and arXiv. +""" + +from __future__ import annotations + +from fable_engine.scrapers.arxiv import ArxivScraper, scrape_arxiv +from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url +from fable_engine.scrapers.github import GitHubScraper, scrape_github +from fable_engine.scrapers.reddit import RedditScraper, scrape_reddit +from fable_engine.scrapers.web import WebScraper, scrape_web +from fable_engine.scrapers.x import XScraper, scrape_x +from fable_engine.scrapers.youtube import YouTubeScraper, scrape_youtube + +__all__ = [ + "ResearchResult", + "ResearchSource", + "fetch_url", + "WebScraper", + "scrape_web", + "YouTubeScraper", + "scrape_youtube", + "RedditScraper", + "scrape_reddit", + "XScraper", + "scrape_x", + "GitHubScraper", + "scrape_github", + "ArxivScraper", + "scrape_arxiv", +] diff --git a/fable_engine/scrapers/arxiv.py b/fable_engine/scrapers/arxiv.py new file mode 100644 index 0000000..0ceb357 --- /dev/null +++ b/fable_engine/scrapers/arxiv.py @@ -0,0 +1,115 @@ +""" +arXiv research paper and search query scraper implementation using arXiv export API. +""" + +from __future__ import annotations + +import re +import urllib.parse +import xml.etree.ElementTree as ET +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url + + +class ArxivScraper(ResearchSource): + source_type = "arxiv" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 10000) -> ResearchResult: + target = target.strip() + if not target: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url="", + error="Target arXiv paper ID or search query cannot be empty." + ) + + paper_id = None + if "arxiv.org/" in target: + id_match = re.search(r"(?:abs|pdf)/([0-9]+\.[0-9]+(?:v[0-9]+)?)", target) + if id_match: + paper_id = id_match.group(1) + elif re.match(r"^[0-9]{4}\.[0-9]{4,5}(?:v[0-9]+)?$", target): + paper_id = target + + if paper_id: + api_url = f"https://export.arxiv.org/api/query?id_list={paper_id}" + canonical_url = f"https://arxiv.org/abs/{paper_id}" + else: + query_enc = urllib.parse.quote(target) + api_url = f"https://export.arxiv.org/api/query?search_query=all:{query_enc}&max_results=5" + canonical_url = f"https://arxiv.org/search/?query={query_enc}&searchtype=all" + + try: + xml_str = fetch_url(api_url, timeout=timeout) + root = ET.fromstring(xml_str) + ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} + + entries = root.findall("atom:entry", ns) + if not entries: + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"arXiv Search: '{target}'", + content="*(No papers found for this query)*" + ) + + blocks = [] + first_title = "" + first_author = "" + for entry in entries: + title_elem = entry.find("atom:title", ns) + title = re.sub(r"\s+", " ", title_elem.text).strip() if title_elem is not None and title_elem.text else "Untitled" + if not first_title: + first_title = title + + id_elem = entry.find("atom:id", ns) + entry_id = id_elem.text.strip() if id_elem is not None and id_elem.text else "" + + published_elem = entry.find("atom:published", ns) + published = published_elem.text[:10] if published_elem is not None and published_elem.text else "" + + summary_elem = entry.find("atom:summary", ns) + summary = re.sub(r"\s+", " ", summary_elem.text).strip() if summary_elem is not None and summary_elem.text else "" + + authors = [] + for author_elem in entry.findall("atom:author", ns): + name_elem = author_elem.find("atom:name", ns) + if name_elem is not None and name_elem.text: + authors.append(name_elem.text.strip()) + if not first_author and authors: + first_author = ", ".join(authors) + + pdf_link = entry_id.replace("/abs/", "/pdf/") + ".pdf" if "/abs/" in entry_id else entry_id + + blocks.append(f"## {title}") + blocks.append(f"**Authors**: {', '.join(authors)}") + blocks.append(f"**Published**: {published} | **arXiv**: [{entry_id}]({entry_id}) | **PDF**: [{pdf_link}]({pdf_link})\n") + blocks.append(f"### Abstract\n{summary}\n") + + full_content = "\n".join(blocks) + if len(full_content) > max_content_length: + full_content = full_content[:max_content_length] + "\n\n*(Abstracts truncated)*" + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=first_title if paper_id else f"arXiv Papers for: '{target}'", + author=first_author, + content=full_content, + metadata={"paper_count": len(entries)} + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"arXiv fetch failed: {str(e)}" + ) + + +def scrape_arxiv(target: str, timeout: int = 15) -> ResearchResult: + return ArxivScraper().fetch(target, timeout=timeout) diff --git a/fable_engine/scrapers/base.py b/fable_engine/scrapers/base.py new file mode 100644 index 0000000..4113269 --- /dev/null +++ b/fable_engine/scrapers/base.py @@ -0,0 +1,386 @@ +""" +Base research scraper interfaces, structured result objects, and HTTP transport utilities. +Enforces standard TLS certificate verification, strict SSRF protection with alternate IP +canonicalization & IP pinning (preventing DNS rebinding TOCTOU window), host rate limiting, +clamped retry/backoff, response bounds, and explicit untrusted external content boundaries. +""" + +from __future__ import annotations + +import datetime +import http.client +import ipaddress +import json +import logging +import re +import socket +import ssl +import threading +import time +import urllib.error +import urllib.parse +from dataclasses import dataclass, field +from html.parser import HTMLParser +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger("fable-engine.scrapers.base") + +USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 FableResearch/1.3.0" + +MAX_RETRY_DELAY_SECONDS = 10.0 + +# Standard TLS context enforcing certificate verification +_SSL_CONTEXT = ssl.create_default_context() + + +def is_safe_ip(ip_str: str) -> bool: + """Verifies that an IP address is public and safe (not loopback, private, link-local, reserved).""" + try: + ip = ipaddress.ip_address(ip_str) + if ( + ip.is_loopback + or ip.is_private + or ip.is_link_local + or ip.is_reserved + or ip.is_multicast + or ip.is_unspecified + ): + return False + # Cloud metadata service check (169.254.169.254) + if str(ip) == "169.254.169.254": + return False + return True + except ValueError: + return False + + +def parse_canonical_ip(hostname_str: str) -> Optional[str]: + """Attempts to parse hostname as standard or alternate IPv4/IPv6 address (hex, octal, short IPv4).""" + try: + return str(ipaddress.ip_address(hostname_str)) + except ValueError: + pass + try: + raw_bytes = socket.inet_aton(hostname_str) + return str(ipaddress.ip_address(raw_bytes)) + except (socket.error, ValueError, OverflowError): + return None + + +def validate_safe_url(url: str) -> Tuple[bool, str, Optional[str]]: + """ + Validates that a URL uses http/https scheme and its hostname resolves + exclusively to safe, public IP addresses (SSRF prevention). + Returns (is_safe, reason, canonical_resolved_ip). + """ + try: + parsed = urllib.parse.urlparse(url) + scheme = parsed.scheme.lower() + if scheme not in ("http", "https"): + return False, f"Invalid URL scheme '{parsed.scheme}'. Strictly http and https are permitted.", None + + hostname = parsed.hostname + if not hostname: + return False, "URL missing valid hostname.", None + + hostname_clean = hostname.strip().lower() + + # Reject explicit local hostnames + if hostname_clean in ("localhost", "localhost.localdomain") or hostname_clean.endswith(".local"): + return False, f"SSRF blocked: local hostname '{hostname_clean}' is not permitted.", None + + # Check if hostname is a standard or alternate numeric IP string (octal, hex, decimal) + canonical_ip = parse_canonical_ip(hostname_clean) + if canonical_ip is not None: + if not is_safe_ip(canonical_ip): + return False, f"SSRF blocked: target IP '{hostname_clean}' ({canonical_ip}) is private, loopback, or reserved.", None + return True, "ok", canonical_ip + + # Resolve hostname DNS records + port = parsed.port or (443 if scheme == "https" else 80) + try: + addr_info = socket.getaddrinfo(hostname_clean, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except socket.gaierror as e: + return False, f"DNS resolution failed for host '{hostname_clean}': {e}", None + + if not addr_info: + return False, f"Could not resolve IP addresses for host '{hostname_clean}'.", None + + safe_ips = [] + for family, _, _, _, sockaddr in addr_info: + ip_str = sockaddr[0] + if not is_safe_ip(ip_str): + return False, f"SSRF blocked: host '{hostname_clean}' resolved to restricted IP '{ip_str}'.", None + safe_ips.append(ip_str) + + return True, "ok", safe_ips[0] + except Exception as e: + return False, f"URL validation error: {e}", None + + +class PinnedHTTPConnection(http.client.HTTPConnection): + """HTTP connection connecting directly to a pre-validated safe IP address.""" + + def __init__(self, host: str, resolved_ip: str, port: int = 80, **kwargs): + super().__init__(host, port=port, **kwargs) + self.resolved_ip = resolved_ip + + def connect(self): + self.sock = socket.create_connection((self.resolved_ip, self.port), self.timeout) + + +class PinnedHTTPSConnection(http.client.HTTPSConnection): + """HTTPS connection connecting directly to a pre-validated safe IP address with TLS SNI.""" + + def __init__(self, host: str, resolved_ip: str, port: int = 443, **kwargs): + super().__init__(host, port=port, **kwargs) + self.resolved_ip = resolved_ip + + def connect(self): + sock = socket.create_connection((self.resolved_ip, self.port), self.timeout) + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + + +class DomainRateLimiter: + """Thread-safe outbound domain rate limiter ensuring minimum spacing between requests.""" + + def __init__(self, min_interval_seconds: float = 0.3): + self.min_interval = min_interval_seconds + self._last_request: Dict[str, float] = {} + self._lock = threading.Lock() + + def wait_if_needed(self, url: str): + try: + hostname = urllib.parse.urlparse(url).hostname or "default" + domain = hostname.lower() + except Exception: + domain = "default" + + with self._lock: + now = time.monotonic() + last = self._last_request.get(domain, 0.0) + elapsed = now - last + if elapsed < self.min_interval: + sleep_time = self.min_interval - elapsed + else: + sleep_time = 0.0 + self._last_request[domain] = now + sleep_time + + if sleep_time > 0: + time.sleep(sleep_time) + + +GLOBAL_RATE_LIMITER = DomainRateLimiter(min_interval_seconds=0.3) + + +@dataclass +class ResearchResult: + """Structured research retrieval result.""" + ok: bool + source_type: str + canonical_url: str + title: str = "" + author: str = "" + content: str = "" + error: Optional[str] = None + retrieved_at: str = field(default_factory=lambda: datetime.datetime.now(datetime.timezone.utc).isoformat()) + metadata: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + if "trust_level" not in self.metadata: + self.metadata["trust_level"] = "untrusted_external_content" + + def to_markdown(self) -> str: + """Formats the structured research result as readable Markdown with explicit untrusted content boundaries.""" + if not self.ok: + return ( + f"# Research Retrieval Failed: {self.source_type.title()}\n" + f"**Canonical URL**: {self.canonical_url or 'N/A'}\n" + f"**Retrieved At**: {self.retrieved_at}\n" + f"**Error**: {self.error or 'Unknown error'}\n" + ) + md = [ + f"# {self.source_type.title()}: {self.title or 'Untitled'}", + f"**Canonical URL**: {self.canonical_url}", + ] + if self.author: + md.append(f"**Author**: {self.author}") + md.append(f"**Retrieved At**: {self.retrieved_at}\n") + if self.content: + md.append("[BEGIN UNTRUSTED EXTERNAL RESEARCH CONTENT]") + md.append(self.content) + md.append("[END UNTRUSTED EXTERNAL RESEARCH CONTENT]") + return "\n".join(md) + + +def fetch_url( + url: str, + headers: Optional[Dict[str, str]] = None, + timeout: int = 15, + max_bytes: int = 5 * 1024 * 1024, + max_retries: int = 2, + backoff_factor: float = 0.5, +) -> str: + """ + Fetches raw string content from URL using pinned IP transport: + 1. Validates host & DNS records (including alternate numeric IPs) for SSRF safety + 2. Pins connection directly to validated IP (immune to DNS rebinding) + 3. Handles HTTP 301/302/303/307/308 redirects with re-validation on every redirect + 4. Applies domain rate limiting + 5. Clamps server Retry-After delays to a hard maximum ceiling (10s) + 6. Standard TLS certificate verification & retries on 429/5xx status + """ + current_url = url + max_redirects = 5 + + for redirect_count in range(max_redirects + 1): + safe, reason, resolved_ip = validate_safe_url(current_url) + if not safe or not resolved_ip: + raise ValueError(f"SSRF validation failed for '{current_url}': {reason}") + + GLOBAL_RATE_LIMITER.wait_if_needed(current_url) + + parsed = urllib.parse.urlparse(current_url) + scheme = parsed.scheme.lower() + host = parsed.hostname + port = parsed.port or (443 if scheme == "https" else 80) + path = parsed.path or "/" + if parsed.query: + path += f"?{parsed.query}" + + req_headers = {"User-Agent": USER_AGENT, "Host": host} + if headers: + req_headers.update(headers) + + last_exc: Optional[Exception] = None + + for attempt in range(max_retries + 1): + try: + if scheme == "https": + conn = PinnedHTTPSConnection(host, resolved_ip, port=port, context=_SSL_CONTEXT, timeout=timeout) + else: + conn = PinnedHTTPConnection(host, resolved_ip, port=port, timeout=timeout) + + conn.request("GET", path, headers=req_headers) + resp = conn.getresponse() + + # Handle HTTP redirects safely + if resp.status in (301, 302, 303, 307, 308): + location = resp.getheader("Location") + conn.close() + if not location: + raise urllib.error.HTTPError(current_url, resp.status, "Redirect missing Location header", resp.headers, None) + current_url = urllib.parse.urljoin(current_url, location) + break # Break retry loop and execute next redirect iteration + + if resp.status in (429, 500, 502, 503, 504) and attempt < max_retries: + retry_after = resp.getheader("Retry-After") + conn.close() + raw_delay = float(retry_after) if retry_after and retry_after.isdigit() else (backoff_factor * (2 ** attempt)) + sleep_time = min(max(0.0, raw_delay), MAX_RETRY_DELAY_SECONDS) + time.sleep(sleep_time) + continue + + if resp.status >= 400: + conn.close() + raise urllib.error.HTTPError(current_url, resp.status, f"HTTP Error {resp.status}", resp.headers, None) + + raw_bytes = resp.read(max_bytes + 1) + conn.close() + if len(raw_bytes) > max_bytes: + raw_bytes = raw_bytes[:max_bytes] + charset = resp.headers.get_param("charset") or "utf-8" + return raw_bytes.decode(charset, errors="replace") + + except urllib.error.HTTPError as exc: + last_exc = exc + if exc.code in (301, 302, 303, 307, 308): + raise exc + if exc.code in (429, 500, 502, 503, 504) and attempt < max_retries: + time.sleep(backoff_factor * (2 ** attempt)) + continue + raise exc + except (urllib.error.URLError, TimeoutError, OSError, http.client.HTTPException) as exc: + last_exc = exc + if attempt < max_retries: + time.sleep(backoff_factor * (2 ** attempt)) + continue + raise exc + + # If loop exited due to redirect, continue outer redirect_count loop + if 'resp' in locals() and resp.status in (301, 302, 303, 307, 308): + continue + + if last_exc: + raise last_exc + + raise RuntimeError(f"Maximum redirects ({max_redirects}) exceeded for {url}") + + +class SimpleHTMLTextExtractor(HTMLParser): + """HTML parser converting web content to readable Markdown text.""" + + def __init__(self): + super().__init__() + self.title = "" + self.in_title = False + self.in_script = False + self.in_style = False + self.chunks: List[str] = [] + self.links: List[Tuple[str, str]] = [] + self._current_href: Optional[str] = None + self._current_link_text: List[str] = [] + + def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]): + tag = tag.lower() + if tag in ("script", "style", "noscript", "svg"): + self.in_script = True + elif tag == "title": + self.in_title = True + elif tag == "a": + attr_dict = dict(attrs) + self._current_href = attr_dict.get("href") + self._current_link_text = [] + elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"): + self.chunks.append("\n\n### ") + elif tag in ("p", "div", "section", "article", "li", "tr"): + self.chunks.append("\n") + + def handle_endtag(self, tag: str): + tag = tag.lower() + if tag in ("script", "style", "noscript", "svg"): + self.in_script = False + elif tag == "title": + self.in_title = False + elif tag == "a": + if self._current_href: + link_str = "".join(self._current_link_text).strip() + if link_str and not self._current_href.startswith("javascript:"): + self.links.append((link_str, self._current_href)) + self._current_href = None + self._current_link_text = [] + + def handle_data(self, data: str): + if self.in_script or self.in_style: + return + if self.in_title: + self.title += data + return + text = data.strip() + if text: + if self._current_href is not None: + self._current_link_text.append(text) + self.chunks.append(text + " ") + + def get_markdown(self) -> str: + raw_text = "".join(self.chunks) + cleaned = re.sub(r"\n\s*\n+", "\n\n", raw_text).strip() + return cleaned + + +class ResearchSource: + """Abstract base class for research scrapers.""" + source_type: str = "generic" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 12000) -> ResearchResult: + raise NotImplementedError diff --git a/fable_engine/scrapers/github.py b/fable_engine/scrapers/github.py new file mode 100644 index 0000000..ef53d4d --- /dev/null +++ b/fable_engine/scrapers/github.py @@ -0,0 +1,129 @@ +""" +GitHub repository metadata, README content, and repository search scraper implementation. +""" + +from __future__ import annotations + +import json +import re +import urllib.parse +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url + + +class GitHubScraper(ResearchSource): + source_type = "github" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 8000) -> ResearchResult: + target = target.strip() + if not target: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url="", + error="Target GitHub repository or search query cannot be empty." + ) + + repo_path = None + if "github.com/" in target: + parsed = urllib.parse.urlparse(target) + parts = [p for p in parsed.path.strip("/").split("/") if p] + if len(parts) >= 2: + repo_path = f"{parts[0]}/{parts[1]}" + elif re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", target): + repo_path = target + + if repo_path: + api_url = f"https://api.github.com/repos/{repo_path}" + canonical_url = f"https://github.com/{repo_path}" + try: + repo_json = fetch_url(api_url, headers={"Accept": "application/vnd.github.v3+json"}, timeout=timeout) + data = json.loads(repo_json) + + name = data.get("full_name", repo_path) + description = data.get("description", "No description provided.") + stars = data.get("stargazers_count", 0) + forks = data.get("forks_count", 0) + language = data.get("language", "Unknown") + default_branch = data.get("default_branch", "main") + html_url = data.get("html_url", canonical_url) + + content_blocks = [ + f"**Stars**: {stars} | **Forks**: {forks} | **Language**: {language} | **Branch**: {default_branch}\n", + f"**Description**: {description}\n" + ] + + # Fetch README content + readme_url = f"https://raw.githubusercontent.com/{repo_path}/{default_branch}/README.md" + try: + readme_text = fetch_url(readme_url, timeout=timeout) + if len(readme_text) > max_content_length: + readme_text = readme_text[:max_content_length] + "\n\n*(README truncated for size)*" + content_blocks.append(f"## README.md\n{readme_text}") + except Exception: + content_blocks.append("*(Note: README.md could not be retrieved automatically)*") + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=html_url, + title=f"GitHub Repo: {name}", + author=repo_path.split("/")[0], + content="\n".join(content_blocks), + metadata={"stars": stars, "language": language} + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"GitHub repository fetch failed: {str(e)}" + ) + + # Repository Search Fallback + query_enc = urllib.parse.quote(target) + search_url = f"https://api.github.com/search/repositories?q={query_enc}&per_page=10" + canonical_url = f"https://github.com/search?q={query_enc}" + try: + search_json = fetch_url(search_url, headers={"Accept": "application/vnd.github.v3+json"}, timeout=timeout) + data = json.loads(search_json) + items = data.get("items", []) + + if not items: + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"GitHub Search: '{target}'", + content="*(No repositories found for this query)*" + ) + + lines = ["## Repositories\n"] + for item in items[:10]: + full_name = item.get("full_name", "") + desc = item.get("description", "") or "No description" + stars = item.get("stargazers_count", 0) + lang = item.get("language", "N/A") + url = item.get("html_url", "") + lines.append(f"### [{full_name}]({url})") + lines.append(f"**Stars**: {stars} | **Language**: {lang}\n**Description**: {desc}\n") + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"GitHub Repository Search: '{target}'", + content="\n".join(lines) + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"GitHub repository search failed: {str(e)}" + ) + + +def scrape_github(target: str, timeout: int = 15) -> ResearchResult: + return GitHubScraper().fetch(target, timeout=timeout) diff --git a/fable_engine/scrapers/reddit.py b/fable_engine/scrapers/reddit.py new file mode 100644 index 0000000..a072ee1 --- /dev/null +++ b/fable_engine/scrapers/reddit.py @@ -0,0 +1,154 @@ +""" +Reddit threads and subreddits scraper implementation using Reddit's public JSON interface (.json). +""" + +from __future__ import annotations + +import json +import urllib.parse +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url + + +class RedditScraper(ResearchSource): + source_type = "reddit" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 10000) -> ResearchResult: + target = target.strip() + if not target: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url="", + error="Target Reddit URL, subreddit, or search query cannot be empty." + ) + + if target.startswith("http://") or target.startswith("https://"): + clean_url = target.split("?")[0].rstrip("/") + url = clean_url + ".json" + canonical_url = clean_url + elif target.startswith("r/") or target.startswith("/r/"): + sub_path = target.lstrip("/").rstrip("/") + url = f"https://www.reddit.com/{sub_path}/hot.json?limit=10" + canonical_url = f"https://www.reddit.com/{sub_path}" + else: + query_enc = urllib.parse.quote(target) + url = f"https://www.reddit.com/search.json?q={query_enc}&limit=10" + canonical_url = f"https://www.reddit.com/search?q={query_enc}" + + try: + json_str = fetch_url(url, timeout=timeout) + data = json.loads(json_str) + + # Handle single thread post response (list of listings) + if isinstance(data, list) and len(data) >= 1 and isinstance(data[0], dict): + children = data[0].get("data", {}).get("children", []) + if not children or not isinstance(children[0], dict): + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error="Reddit post data structure was empty or unexpected." + ) + + post_data = children[0].get("data", {}) + title = post_data.get("title", "Reddit Post") + subreddit = post_data.get("subreddit", "") + author = post_data.get("author", "[deleted]") + score = post_data.get("score", 0) + selftext = post_data.get("selftext", "") + permalink = post_data.get("permalink", "") + post_url = f"https://www.reddit.com{permalink}" if permalink else canonical_url + + content_blocks = [ + f"**Subreddit**: r/{subreddit} | **Author**: u/{author} | **Score**: {score}\n" + ] + if selftext: + content_blocks.append(f"## Post Text\n{selftext}\n") + + if len(data) >= 2 and isinstance(data[1], dict): + comments = data[1].get("data", {}).get("children", []) + comment_lines = ["## Top Comments\n"] + count = 0 + for c in comments: + if count >= 10: + break + if isinstance(c, dict): + cdata = c.get("data", {}) + c_author = cdata.get("author", "") + c_body = cdata.get("body", "") + c_score = cdata.get("score", 0) + if c_body: + comment_lines.append(f"### u/{c_author} ({c_score} points)") + comment_lines.append(f"{c_body}\n") + count += 1 + if count > 0: + content_blocks.extend(comment_lines) + + full_content = "\n".join(content_blocks) + if len(full_content) > max_content_length: + full_content = full_content[:max_content_length] + "\n\n*(Content truncated)*" + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=post_url, + title=title, + author=f"u/{author}", + content=full_content, + metadata={"subreddit": subreddit, "score": score} + ) + + # Handle subreddit listing or search results + elif isinstance(data, dict) and "data" in data: + children = data.get("data", {}).get("children", []) + if not children: + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"Reddit Listing: '{target}'", + content="*(No posts found for this subreddit or query)*" + ) + + items = ["## Posts\n"] + for child in children[:10]: + if isinstance(child, dict): + p = child.get("data", {}) + p_title = p.get("title", "") + p_sub = p.get("subreddit", "") + p_author = p.get("author", "") + p_score = p.get("score", 0) + p_comments = p.get("num_comments", 0) + p_permalink = f"https://www.reddit.com{p.get('permalink', '')}" + p_text = p.get("selftext", "")[:250] + text_str = f"\n _{p_text.strip()}..._" if p_text else "" + items.append(f"- **[{p_title}]({p_permalink})**") + items.append(f" *r/{p_sub} | u/{p_author} | Score: {p_score} | Comments: {p_comments}*{text_str}\n") + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"Reddit Listing: '{target}'", + content="\n".join(items) + ) + + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error="Unexpected JSON response format from Reddit API." + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"Reddit fetch failed: {str(e)}" + ) + + +def scrape_reddit(target: str, timeout: int = 15) -> ResearchResult: + return RedditScraper().fetch(target, timeout=timeout) diff --git a/fable_engine/scrapers/web.py b/fable_engine/scrapers/web.py new file mode 100644 index 0000000..6defa43 --- /dev/null +++ b/fable_engine/scrapers/web.py @@ -0,0 +1,120 @@ +""" +Web page and search query scraper implementation. +""" + +from __future__ import annotations + +import re +import urllib.parse +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, SimpleHTMLTextExtractor, fetch_url + + +class WebScraper(ResearchSource): + source_type = "web" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 12000) -> ResearchResult: + target = target.strip() + if not target: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url="", + error="Target URL or query string cannot be empty." + ) + + if target.startswith("http://") or target.startswith("https://"): + try: + html = fetch_url(target, timeout=timeout) + parser = SimpleHTMLTextExtractor() + parser.feed(html) + title = parser.title.strip() or target + content = parser.get_markdown() + if len(content) > max_content_length: + content = content[:max_content_length] + "\n\n*(Content truncated for size)*" + + links_md = "" + if parser.links: + seen = set() + link_lines = [] + for text, href in parser.links[:15]: + full_url = urllib.parse.urljoin(target, href) + if full_url not in seen and text: + seen.add(full_url) + link_lines.append(f"- [{text}]({full_url})") + if link_lines: + links_md = "\n\n## Key Links\n" + "\n".join(link_lines) + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=target, + title=title, + content=f"## Web Page Content\n{content}{links_md}" + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=target, + error=f"Web page fetch failed: {str(e)}" + ) + else: + # DuckDuckGo HTML Search + try: + query_enc = urllib.parse.quote(target) + ddg_url = f"https://html.duckduckgo.com/html/?q={query_enc}" + html = fetch_url(ddg_url, timeout=timeout) + + results = [] + result_blocks = re.findall( + r'.*?(.*?)', + html, + re.DOTALL + ) + if not result_blocks: + urls = re.findall(r'class="result__a" href="([^"]+)">([^<]+)', html) + snippets = re.findall(r'class="result__snippet[^"]*">(.*?)', html) + for i in range(min(len(urls), 10)): + raw_url, title = urls[i] + parsed_url = urllib.parse.parse_qs(urllib.parse.urlparse(raw_url).query).get("uddg", [raw_url])[0] + snip = snippets[i] if i < len(snippets) else "" + snip_clean = re.sub(r"<[^>]+>", "", snip).strip() + title_clean = re.sub(r"<[^>]+>", "", title).strip() + results.append(f"### [{title_clean}]({parsed_url})\n**Snippet**: {snip_clean}\n") + else: + for raw_url, snip in result_blocks[:10]: + snip_clean = re.sub(r"<[^>]+>", "", snip).strip() + results.append(f"### [Result]({raw_url})\n**Snippet**: {snip_clean}\n") + + if not results: + parser = SimpleHTMLTextExtractor() + parser.feed(html) + content = parser.get_markdown()[:max_content_length] + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=ddg_url, + title=f"Web Search: '{target}'", + content=content + ) + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=ddg_url, + title=f"Web Search Results for: '{target}'", + content="\n".join(results) + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=f"https://duckduckgo.com/html/?q={urllib.parse.quote(target)}", + error=f"Web search failed: {str(e)}" + ) + + +def scrape_web(target: str, timeout: int = 15) -> ResearchResult: + return WebScraper().fetch(target, timeout=timeout) diff --git a/fable_engine/scrapers/x.py b/fable_engine/scrapers/x.py new file mode 100644 index 0000000..ca71687 --- /dev/null +++ b/fable_engine/scrapers/x.py @@ -0,0 +1,125 @@ +""" +X (Twitter) tweet syndication and oembed lookup scraper implementation. +Exact Tweet URLs utilize Twitter syndication API; handles/profiles use oEmbed embed summaries. +""" + +from __future__ import annotations + +import json +import re +import urllib.parse +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, SimpleHTMLTextExtractor, fetch_url + + +class XScraper(ResearchSource): + source_type = "x" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 8000) -> ResearchResult: + target = target.strip() + if not target: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url="", + error="Target Tweet URL or handle cannot be empty." + ) + + tweet_id = None + tweet_id_match = re.search(r"status(?:es)?/(\d+)", target) + if tweet_id_match: + tweet_id = tweet_id_match.group(1) + + if tweet_id: + # Use Twitter Syndication JSON API (free endpoint for individual tweets) + synd_url = f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}&token=x" + canonical_url = f"https://x.com/i/status/{tweet_id}" + try: + json_str = fetch_url(synd_url, timeout=timeout) + data = json.loads(json_str) + + text = data.get("text", "") + user = data.get("user", {}) + name = user.get("name", "User") + screen_name = user.get("screen_name", "unknown") + created_at = data.get("created_at", "") + likes = data.get("favorite_count", 0) + retweets = data.get("retweet_count", 0) + full_url = f"https://x.com/{screen_name}/status/{tweet_id}" + + content = ( + f"**Date**: {created_at} | **Likes**: {likes} | **Retweets**: {retweets}\n\n" + f"## Tweet\n{text}" + ) + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=full_url, + title=f"Tweet by @{screen_name} ({name})", + author=f"@{screen_name}", + content=content, + metadata={"tweet_id": tweet_id, "likes": likes, "retweets": retweets} + ) + except Exception as e: + # Fallback to Twitter oEmbed API + try: + oembed_url = f"https://publish.twitter.com/oembed?url=https://twitter.com/i/status/{tweet_id}" + json_str = fetch_url(oembed_url, timeout=timeout) + data = json.loads(json_str) + author = data.get("author_name", "User") + html_embed = data.get("html", "") + parser = SimpleHTMLTextExtractor() + parser.feed(html_embed) + text = parser.get_markdown() + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"Tweet by {author}", + author=author, + content=f"## Content\n{text}" + ) + except Exception: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"X/Twitter tweet scrape failed for status ID {tweet_id}: {str(e)}" + ) + + # Handle user profile / handle lookup via oembed + handle = target.lstrip("@").strip() + canonical_url = f"https://x.com/{handle}" + try: + oembed_url = f"https://publish.twitter.com/oembed?url=https://twitter.com/{handle}" + json_str = fetch_url(oembed_url, timeout=timeout) + data = json.loads(json_str) + author = data.get("author_name", handle) + html_embed = data.get("html", "") + parser = SimpleHTMLTextExtractor() + parser.feed(html_embed) + text = parser.get_markdown() + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=f"X Profile / Handle: @{handle}", + author=author, + content=f"## Summary\n{text}" + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=( + f"X handle/profile lookup failed for '@{handle}': {str(e)}. " + "Note: For exact tweet contents, provide a direct Tweet status URL." + ) + ) + + +def scrape_x(target: str, timeout: int = 15) -> ResearchResult: + return XScraper().fetch(target, timeout=timeout) diff --git a/fable_engine/scrapers/youtube.py b/fable_engine/scrapers/youtube.py new file mode 100644 index 0000000..afddd80 --- /dev/null +++ b/fable_engine/scrapers/youtube.py @@ -0,0 +1,132 @@ +""" +Best-effort YouTube metadata, search, and caption track scraper implementation. +Uses heuristic page-structure parsing for public watch page metadata and automated caption tracks. +""" + +from __future__ import annotations + +import json +import re +import urllib.parse +import xml.etree.ElementTree as ET +from typing import Optional + +from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url + + +class YouTubeScraper(ResearchSource): + """ + Best-effort scraper for YouTube public metadata and caption tracks. + Heuristic page parsing is subject to upstream YouTube markup variations. + """ + source_type = "youtube" + + def fetch(self, target: str, timeout: int = 15, max_content_length: int = 8000) -> ResearchResult: + target = target.strip() + video_id = None + + if "youtube.com" in target or "youtu.be" in target: + parsed = urllib.parse.urlparse(target) + if "youtu.be" in target: + video_id = parsed.path.lstrip("/") + else: + qs = urllib.parse.parse_qs(parsed.query) + video_id = qs.get("v", [None])[0] + elif len(target) == 11 and re.match(r"^[A-Za-z0-9_-]{11}$", target): + video_id = target + + if not video_id: + # Best-effort search resolution + try: + query_enc = urllib.parse.quote(target) + search_url = f"https://www.youtube.com/results?search_query={query_enc}" + html = fetch_url(search_url, timeout=timeout) + vids = re.findall(r'"videoId":"([A-Za-z0-9_-]{11})"', html) + if vids: + video_id = vids[0] + else: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=search_url, + error=f"Best-effort YouTube search could not resolve video ID for '{target}'." + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=f"https://www.youtube.com/results?search_query={urllib.parse.quote(target)}", + error=f"Best-effort YouTube search resolution failed: {str(e)}" + ) + + canonical_url = f"https://www.youtube.com/watch?v={video_id}" + try: + html = fetch_url(canonical_url, timeout=timeout) + + title_match = re.search(r' max_content_length: + transcript_text = transcript_text[:max_content_length] + "...\n*(Transcript truncated)*" + content_blocks.append(f"## Transcript / Captions\n{transcript_text}") + else: + content_blocks.append(f"*(Note: Best-effort automated transcript track status: {transcript_status})*") + + return ResearchResult( + ok=True, + source_type=self.source_type, + canonical_url=canonical_url, + title=title, + author=channel, + content="\n".join(content_blocks), + metadata={ + "video_id": video_id, + "extraction_mode": "best_effort_heuristic", + "transcript_status": transcript_status, + } + ) + except Exception as e: + return ResearchResult( + ok=False, + source_type=self.source_type, + canonical_url=canonical_url, + error=f"Best-effort YouTube video scrape failed: {str(e)}" + ) + + +def scrape_youtube(target: str, timeout: int = 15) -> ResearchResult: + return YouTubeScraper().fetch(target, timeout=timeout) diff --git a/fable_mode/manifest.py b/fable_mode/manifest.py index bc5568a..d95040e 100644 --- a/fable_mode/manifest.py +++ b/fable_mode/manifest.py @@ -22,8 +22,17 @@ "fable_engine/actions/deliberation.py", "fable_engine/actions/fleet.py", "fable_engine/actions/lifecycle.py", + "fable_engine/actions/scrapers.py", "fable_engine/actions/system3.py", "fable_engine/cas.py", + "fable_engine/scrapers/__init__.py", + "fable_engine/scrapers/arxiv.py", + "fable_engine/scrapers/base.py", + "fable_engine/scrapers/github.py", + "fable_engine/scrapers/reddit.py", + "fable_engine/scrapers/web.py", + "fable_engine/scrapers/x.py", + "fable_engine/scrapers/youtube.py", "fable_engine/guards.py", "fable_engine/schema.py", "fable_engine/server.py", diff --git a/fable_mode/resources.json b/fable_mode/resources.json index 1ed64ed..102ef08 100644 --- a/fable_mode/resources.json +++ b/fable_mode/resources.json @@ -15,8 +15,17 @@ "fable_engine/actions/deliberation.py", "fable_engine/actions/fleet.py", "fable_engine/actions/lifecycle.py", + "fable_engine/actions/scrapers.py", "fable_engine/actions/system3.py", "fable_engine/cas.py", + "fable_engine/scrapers/__init__.py", + "fable_engine/scrapers/arxiv.py", + "fable_engine/scrapers/base.py", + "fable_engine/scrapers/github.py", + "fable_engine/scrapers/reddit.py", + "fable_engine/scrapers/web.py", + "fable_engine/scrapers/x.py", + "fable_engine/scrapers/youtube.py", "fable_engine/guards.py", "fable_engine/schema.py", "fable_engine/server.py", diff --git a/skills/fable-mode/SKILL.md b/skills/fable-mode/SKILL.md index d278d38..ffc2aad 100644 --- a/skills/fable-mode/SKILL.md +++ b/skills/fable-mode/SKILL.md @@ -10,7 +10,8 @@ description: >- Omniscient Session Lineage & Working Memory (track_file_change, get_session_lineage, inspect_plan, verify_proof, record_visual_mockups), Pre-Flight Goal Score & Rubric Pointers (S_target >= 95%), Autonomous Tool & Pipeline Synthesis ("Automate What Can Be Automated"), 10-Tool Coder Subagent MCP - Fleet (fable_v2.coder_fleet), and dedicated Domain Cognitive Gears (including domain: "design" / + Fleet (fable_v2.coder_fleet), Zero-Cost Multi-Source Research Scraping Engine (`scrape_web`, `scrape_youtube`, + `scrape_reddit`, `scrape_x`, `scrape_github`, `scrape_arxiv`), and dedicated Domain Cognitive Gears (including domain: "design" / Cinematic Design Engine). Enforces strict cognitive separation: Main Agent performs all DeepThink, System 2 deliberation, epistemic verification, and architectural blueprinting (cannot code directly); all code writing and file modifications are executed strictly by subagents. Activates whenever the user @@ -55,6 +56,7 @@ description: >- 20. **Modular Fable Part 2: Dynamic Customizable Cortical Lobe Sprouting & BCM Plasticity (`HebbianPlasticityEngine`)**: Lifelong neuro-evolutionary adaptation replacing static vector RAG with living, customizable cortical domain lobes. Strictly avoids hardcoded domain constraints: dynamically sprouts, names, describes, and evolves custom lobes from scratch for any language, framework, or scientific discipline (e.g. `zig_systems`, `mojo_kernels`, `solidity_evm`, `bioinformatics`) alongside the 5 foundational lobes (`rust`, `python`, `design_3d`, `research`, `concurrency`). Updates synaptic weights via directional BCM / STDP plasticity (LTP on success $\Delta W = +\eta \cdot A_{domain} \cdot A_{j}$, LTD on failure $\Delta W = -\lambda \cdot A_{domain} \cdot A_{j}$) with continuous activity signals, homeostatic normalization bounded in $[0.05, 1.0]$, automated closed-loop RedTeam-to-Cortex consolidation, and immunological `HeuristicAntibody` synthesis. 21. **Autonomous Upstream Synchronization (Rule 19: Autonomous Silent Self-Updater)**: Fable-Mode instances autonomously and silently check for upstream commits and hot-sync themselves across client machines, ensuring every computer is always running the latest version without manual user intervention. Operates via non-blocking daemon threads (0ms session startup delay) and non-destructively preserves local cortical experience (antibodies, synaptic weights, custom lobes). 22. **Anti-Slop Frontend & Awwwards Design Standard (`DesignEngine` & `AntiSlopAuditor`)**: Permanent elimination of AI slop. Even the simplest prompts undergo Brief Inference ("Read the Room") emitting the mandatory Design Read Declaration, calibrating the Three Dials (Variance, Motion, Density), selecting from the 6 Haute Aesthetic Archetypes (Cyber Obsidian Monolith, Haute Editorial, Swiss Precision Vignelli, Kinetic Spatial HUD, Neo-Nordic Warmth, Cold Chromatic Luxury), generating OKLCH colorways, and passing the mechanical 5-Point Pre-Flight Design Quality Gate (100dvh viewport stability, descender clearance, invariant locks, WCAG AA contrast, and zero LLM buzzwords). +23. **Zero-Cost Multi-Source Research Scraping Engine**: Master research capabilities enabling agents to scrape data across 6 primary intelligence platforms (**Web**, **YouTube**, **Reddit**, **X / Twitter**, **GitHub**, and **arXiv**) using 100% free, stdlib-based scrapers with zero API fees. Scraping results return structured Markdown and automatically enter the Epistemic Ledger as candidate `[HYPOTHESIS]` items using `auto_log_epistemic: true` (following the Fable research lifecycle: *retrieve -> normalize -> cite -> cross-check -> promote evidence*). -------------------------------------------------------------------------------- diff --git a/skills/fable-mode/cortex/research.md b/skills/fable-mode/cortex/research.md index 77d9526..bb5956a 100644 --- a/skills/fable-mode/cortex/research.md +++ b/skills/fable-mode/cortex/research.md @@ -11,6 +11,7 @@ synaptic_weights: epistemic_grounding: 0.95 latex_derivation: 0.87 triz_dialectic: 0.89 + zero_cost_scrapers: 0.70 antibodies: - antibody_id: ab_research_hallucinated_citation domain: research @@ -63,6 +64,9 @@ specialized_heuristics: backed), [HYPOTHESIS] (untested inference), or [UNKNOWN] (unmeasured parameter).' - 'Causal vs Correlative Discipline: Construct a Pearlian Directed Acyclic Graph (DAG) before claiming intervention efficacy; rule out confounding and collider bias.' +- 'Zero-Cost Multi-Source Research Scraping: Query scrape_web, scrape_youtube, + scrape_reddit, scrape_x, scrape_github, and scrape_arxiv with auto_log_epistemic: true + to retrieve candidate research evidence [HYPOTHESIS] and execute systematic cross-source verification before promoting to [PROVEN].' last_consolidated_at: '2026-09-04T12:00:00+00:00' --- @@ -95,6 +99,7 @@ last_consolidated_at: '2026-09-04T12:00:00+00:00' | `epistemic_grounding` | `0.9500` | 🟢 Strong | | `first_principles_synthesis` | `0.9400` | 🟢 Strong | | `contradiction_resolver` | `0.9100` | 🟢 Strong | +| `zero_cost_scrapers` | `0.7000` | 🟡 Moderate | | `triz_dialectic` | `0.8900` | 🟢 Strong | | `latex_derivation` | `0.8700` | 🟢 Strong | diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py new file mode 100644 index 0000000..b4cc678 --- /dev/null +++ b/tests/test_scrapers.py @@ -0,0 +1,370 @@ +""" +Unit tests for Fable Engine Zero-Cost Research Scrapers, SSRF protection, +alternate numeric IP canonicalization, redirect validation, IP pinning, +clamped retry delays, content boundaries, error handling, and MCP Action Handlers. +""" + +import json +import unittest +import urllib.error +from unittest.mock import MagicMock, patch + +from fable_engine.actions import handle_fable_session +from fable_engine.scrapers import ( + ArxivScraper, + GitHubScraper, + RedditScraper, + ResearchResult, + WebScraper, + XScraper, + YouTubeScraper, + fetch_url, + scrape_arxiv, + scrape_github, + scrape_reddit, + scrape_web, + scrape_x, + scrape_youtube, +) +from fable_engine.scrapers.base import PinnedHTTPSConnection, parse_canonical_ip, validate_safe_url +from fable_engine.session import get_or_load_session + + +class TestSSRFProtectionAndRateLimiting(unittest.TestCase): + + def test_alternate_numeric_ip_canonicalization(self): + self.assertEqual(parse_canonical_ip("127.0.0.1"), "127.0.0.1") + self.assertEqual(parse_canonical_ip("2130706433"), "127.0.0.1") + self.assertEqual(parse_canonical_ip("127.1"), "127.0.0.1") + self.assertEqual(parse_canonical_ip("0x7f000001"), "127.0.0.1") + self.assertEqual(parse_canonical_ip("0177.0.0.1"), "127.0.0.1") + self.assertIsNone(parse_canonical_ip("arxiv.org")) + + def test_ssrf_validation_blocks_unsafe_hosts_and_alternate_ips(self): + unsafe_urls = [ + "http://localhost:8080/admin", + "http://127.0.0.1/secret", + "http://127.0.0.255/", + "http://2130706433/secret", + "http://127.1/admin", + "http://0x7f000001/status", + "http://0177.0.0.1/keys", + "http://10.0.0.1/internal", + "http://172.16.0.1/status", + "http://192.168.1.1/router", + "http://169.254.169.254/latest/meta-data/", + "file:///etc/passwd", + "gopher://localhost:70", + "ftp://ftp.local/data", + ] + for url in unsafe_urls: + safe, reason, resolved_ip = validate_safe_url(url) + self.assertFalse(safe, f"Expected '{url}' to be blocked by SSRF validation, but it passed.") + + def test_ssrf_validation_allows_public_urls(self): + safe_urls = [ + "https://arxiv.org/abs/2401.12345", + "https://api.github.com/repos/REX-codebase/fable-mode", + "https://www.reddit.com/r/Python/hot.json", + ] + for url in safe_urls: + safe, reason, resolved_ip = validate_safe_url(url) + self.assertTrue(safe, f"Expected public URL '{url}' to pass SSRF validation, failed: {reason}") + self.assertTrue(resolved_ip is not None) + + def test_fetch_url_raises_value_error_on_ssrf_target(self): + with self.assertRaises(ValueError) as ctx: + fetch_url("http://127.0.0.1:8000/keys") + self.assertIn("SSRF validation failed", str(ctx.exception)) + + @patch("fable_engine.scrapers.base.validate_safe_url") + def test_ssrf_redirect_to_private_ip_is_blocked(self, mock_validate): + mock_validate.side_effect = [ + (True, "ok", "93.184.216.34"), # example.com (public) + (False, "SSRF blocked: target IP '127.0.0.1' is private, loopback, or reserved.", None) + ] + + with patch("fable_engine.scrapers.base.PinnedHTTPSConnection") as mock_conn_cls: + mock_conn = MagicMock() + mock_resp_redirect = MagicMock() + mock_resp_redirect.status = 302 + mock_resp_redirect.getheader.side_effect = lambda h: "http://127.0.0.1/secret" if h == "Location" else None + mock_conn.getresponse.return_value = mock_resp_redirect + mock_conn_cls.return_value = mock_conn + + with self.assertRaises(ValueError) as ctx: + fetch_url("https://example.com/redirect-me") + self.assertIn("SSRF validation failed", str(ctx.exception)) + self.assertIn("127.0.0.1", str(ctx.exception)) + + +class TestResearchResultAndContentBoundaries(unittest.TestCase): + + def test_untrusted_content_boundary_formatting(self): + res = ResearchResult( + ok=True, + source_type="web", + canonical_url="https://example.com", + title="Sample Page", + content="Sample webpage text." + ) + self.assertEqual(res.metadata.get("trust_level"), "untrusted_external_content") + md = res.to_markdown() + self.assertIn("[BEGIN UNTRUSTED EXTERNAL RESEARCH CONTENT]", md) + self.assertIn("Sample webpage text.", md) + self.assertIn("[END UNTRUSTED EXTERNAL RESEARCH CONTENT]", md) + + +class TestResearchScrapers(unittest.TestCase): + + @patch("fable_engine.scrapers.web.fetch_url") + def test_scrape_web_url_success(self, mock_fetch): + mock_fetch.return_value = """ + + Test Page + +

Hello World

+

This is a test paragraph with a link.

+ + + """ + res = scrape_web("https://example.com/page") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "web") + self.assertEqual(res.canonical_url, "https://example.com/page") + self.assertIn("Test Page", res.title) + md = res.to_markdown() + self.assertIn("Hello World", md) + self.assertIn("[link](https://example.com/link)", md) + + @patch("fable_engine.scrapers.web.fetch_url") + def test_scrape_web_http_error_handling(self, mock_fetch): + mock_fetch.side_effect = urllib.error.HTTPError("https://example.com/forbidden", 403, "Forbidden", {}, None) + res = scrape_web("https://example.com/forbidden") + self.assertFalse(res.ok) + self.assertIn("403", res.error) + md = res.to_markdown() + self.assertIn("Research Retrieval Failed", md) + + @patch("fable_engine.scrapers.youtube.fetch_url") + def test_scrape_youtube_video_success(self, mock_fetch): + def side_effect(url, *args, **kwargs): + if "watch?v=" in url: + return """ + + + + + "captionTracks":[{"baseUrl":"https://youtube.com/api/timedtext"}] + + """ + elif "timedtext" in url: + return """ + + Welcome to the video. + Today we discuss AI agents. + + """ + return "" + + mock_fetch.side_effect = side_effect + res = scrape_youtube("https://www.youtube.com/watch?v=dQw4w9WgXcQ") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "youtube") + self.assertEqual(res.canonical_url, "https://www.youtube.com/watch?v=dQw4w9WgXcQ") + self.assertIn("Test AI Video", res.title) + self.assertIn("Welcome to the video. Today we discuss AI agents.", res.content) + + @patch("fable_engine.scrapers.youtube.fetch_url") + def test_scrape_youtube_missing_transcript_fallback(self, mock_fetch): + mock_fetch.return_value = """ + + + + + """ + res = scrape_youtube("dQw4w9WgXcQ") + self.assertTrue(res.ok) + self.assertIn("Best-effort automated transcript track status", res.content) + + @patch("fable_engine.scrapers.reddit.fetch_url") + def test_scrape_reddit_thread_success(self, mock_fetch): + mock_response = [ + { + "data": { + "children": [ + { + "data": { + "title": "Fable Engine Released", + "subreddit": "Python", + "author": "rex_dev", + "score": 150, + "selftext": "Check out the new cognitive runtime.", + "permalink": "/r/Python/comments/123/fable/" + } + } + ] + } + }, + { + "data": { + "children": [ + { + "data": { + "author": "user1", + "body": "Looks amazing!", + "score": 25 + } + } + ] + } + } + ] + mock_fetch.return_value = json.dumps(mock_response) + res = scrape_reddit("https://www.reddit.com/r/Python/comments/123/fable/") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "reddit") + self.assertIn("Fable Engine Released", res.title) + self.assertIn("Check out the new cognitive runtime.", res.content) + self.assertIn("u/user1 (25 points)", res.content) + + @patch("fable_engine.scrapers.reddit.fetch_url") + def test_scrape_reddit_empty_structure_handling(self, mock_fetch): + mock_fetch.return_value = json.dumps([]) + res = scrape_reddit("r/empty_sub") + self.assertFalse(res.ok) + self.assertTrue(res.error is not None) + + @patch("fable_engine.scrapers.x.fetch_url") + def test_scrape_x_tweet_success(self, mock_fetch): + mock_tweet = { + "text": "Fable Mode is now 100% free with research scrapers!", + "user": { + "name": "Fable Engine", + "screen_name": "fable_mode" + }, + "created_at": "Mon Mar 03 12:00:00 +0000 2025", + "favorite_count": 42, + "retweet_count": 10 + } + mock_fetch.return_value = json.dumps(mock_tweet) + res = scrape_x("https://x.com/fable_mode/status/1234567890") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "x") + self.assertIn("Fable Engine", res.title) + self.assertIn("100% free", res.content) + + @patch("fable_engine.scrapers.github.fetch_url") + def test_scrape_github_repo_success(self, mock_fetch): + def side_effect(url, *args, **kwargs): + if "api.github.com/repos" in url: + return json.dumps({ + "full_name": "REX-codebase/fable-mode", + "description": "Control plane for AI agents", + "stargazers_count": 100, + "forks_count": 12, + "language": "Python", + "default_branch": "main", + "html_url": "https://github.com/REX-codebase/fable-mode" + }) + elif "raw.githubusercontent.com" in url: + return "# Fable Mode\nStructured Deliberation for Agents." + return "" + + mock_fetch.side_effect = side_effect + res = scrape_github("REX-codebase/fable-mode") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "github") + self.assertIn("REX-codebase/fable-mode", res.title) + self.assertIn("Control plane for AI agents", res.content) + + @patch("fable_engine.scrapers.arxiv.fetch_url") + def test_scrape_arxiv_paper_success(self, mock_fetch): + mock_xml = """ + + + http://arxiv.org/abs/2401.12345v1 + 2024-01-20T10:00:00Z + Cognitive Deliberation in AI Agents + We propose Fable Mode for structured deliberation. + Alice Smith + Bob Jones + + + """ + mock_fetch.return_value = mock_xml + res = scrape_arxiv("2401.12345") + self.assertTrue(res.ok) + self.assertEqual(res.source_type, "arxiv") + self.assertIn("Cognitive Deliberation in AI Agents", res.content) + + +class TestScraperActionsAndEpistemicLog(unittest.TestCase): + + @patch("fable_engine.actions.scrapers.scrape_arxiv") + def test_mcp_action_dispatch(self, mock_scrape): + mock_scrape.return_value = ResearchResult( + ok=True, + source_type="arxiv", + canonical_url="https://arxiv.org/abs/2401.12345", + title="AI Deliberation", + content="Paper abstract." + ) + resp = handle_fable_session({ + "action": "scrape_arxiv", + "target": "2401.12345" + }) + self.assertIn("AI Deliberation", resp) + mock_scrape.assert_called_once_with("2401.12345") + + @patch("fable_engine.actions.scrapers.scrape_github") + def test_auto_log_epistemic_hypothesis_success_only(self, mock_scrape): + session_name = "test_scraper_session" + + handle_fable_session({ + "action": "create_session", + "session_name": session_name, + "objective": "Test scraping auto-log as HYPOTHESIS", + "time_budget_minutes": 2.0 + }) + + # 1. Test failed result (ok=False) -> MUST NOT log hypothesis + mock_scrape.return_value = ResearchResult( + ok=False, + source_type="github", + canonical_url="https://github.com/invalid/repo", + error="404 Not Found" + ) + handle_fable_session({ + "action": "scrape_github", + "session_name": session_name, + "target": "invalid/repo", + "auto_log_epistemic": True + }) + + session = get_or_load_session(session_name) + self.assertEqual(len(session.epistemic_ledger), 0) + + # 2. Test successful result (ok=True) -> MUST log hypothesis + mock_scrape.return_value = ResearchResult( + ok=True, + source_type="github", + canonical_url="https://github.com/REX-codebase/fable-mode", + title="REX-codebase/fable-mode", + content="# GitHub Repo\nName: fable-mode" + ) + handle_fable_session({ + "action": "scrape_github", + "session_name": session_name, + "target": "REX-codebase/fable-mode", + "auto_log_epistemic": True + }) + + session = get_or_load_session(session_name) + hypothesis_items = [item for item in session.epistemic_ledger if item["tag"] == "HYPOTHESIS"] + self.assertEqual(len(hypothesis_items), 1) + self.assertIn("REX-codebase/fable-mode", hypothesis_items[0]["claim"]) + + +if __name__ == "__main__": + unittest.main()