Add Zero-Cost Multi-Source Research Scrapers to Fable Mode

README.md

@@ -257,6 +257,12 @@ The `fable-engine` server exposes the unified `fable_session` tool adhering to J
 | `unlock_execution` | Gatekeeper | `session_name`, `rationale` | Evaluates time-lock clock and refinement criteria to unlock workspace modifications. |
 | `check_auto_update` | Sync | `session_name?` | Probes upstream repository for updates without blocking execution. |
 | `apply_auto_update` | Sync | `preserve_cortex?` | Updates skills and configuration while preserving local cortical learning. |
+| `scrape_web` | Research | `target`, `auto_log_epistemic?` | Scrapes web page URL or performs free DuckDuckGo web search returning Markdown. |
+| `scrape_youtube` | Research | `target`, `auto_log_epistemic?` | Scrapes YouTube video metadata and public caption/transcript tracks returning Markdown. |
+| `scrape_reddit` | Research | `target`, `auto_log_epistemic?` | Scrapes Reddit threads, comments, or subreddits via free JSON API returning Markdown. |
+| `scrape_x` | Research | `target`, `auto_log_epistemic?` | Scrapes individual Tweet URLs via Twitter syndication API and best-effort handle lookups returning Markdown. |
+| `scrape_github` | Research | `target`, `auto_log_epistemic?` | Scrapes GitHub repository metadata, READMEs, or repository search returning Markdown. |
+| `scrape_arxiv` | Research | `target`, `auto_log_epistemic?` | Scrapes arXiv academic papers, abstracts, and PDF links returning Markdown. |
 
 ---
 

fable_engine/actions/__init__.py

@@ -70,6 +70,14 @@
     _handle_validate_preflight_design,
     _handle_list_design_archetypes,
 )
+from fable_engine.actions.scrapers import (
+    _handle_scrape_web,
+    _handle_scrape_youtube,
+    _handle_scrape_reddit,
+    _handle_scrape_x,
+    _handle_scrape_github,
+    _handle_scrape_arxiv,
+)
 
 ACTION_DISPATCH: Dict[str, Callable[[Dict[str, Any]], str]] = {
     "create_session": _handle_create_session,
@@ -236,6 +244,19 @@
     "design_archetypes": _handle_list_design_archetypes,
     "list_archetypes": _handle_list_design_archetypes,
     "archetypes": _handle_list_design_archetypes,
+    "scrape_web": _handle_scrape_web,
+    "web_scrape": _handle_scrape_web,
+    "scrape_youtube": _handle_scrape_youtube,
+    "youtube_scrape": _handle_scrape_youtube,
+    "scrape_reddit": _handle_scrape_reddit,
+    "reddit_scrape": _handle_scrape_reddit,
+    "scrape_x": _handle_scrape_x,
+    "x_scrape": _handle_scrape_x,
+    "twitter_scrape": _handle_scrape_x,
+    "scrape_github": _handle_scrape_github,
+    "github_scrape": _handle_scrape_github,
+    "scrape_arxiv": _handle_scrape_arxiv,
+    "arxiv_scrape": _handle_scrape_arxiv,
 }
 
 

fable_engine/actions/scrapers.py

@@ -0,0 +1,116 @@
+"""
+Fable Engine Research Scraping Action Handlers.
+Exposes zero-cost scrapers for Web, YouTube, Reddit, X, GitHub, and arXiv as individual MCP actions.
+Integrates retrieved evidence into Fable Session epistemic ledger as [HYPOTHESIS] candidate items.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict
+
+from fable_engine.scrapers import (
+    ResearchResult,
+    scrape_arxiv,
+    scrape_github,
+    scrape_reddit,
+    scrape_web,
+    scrape_x,
+    scrape_youtube,
+)
+from fable_engine.session import ACTIVE_SESSIONS, get_or_load_session
+
+logger = logging.getLogger("fable-engine.actions.scrapers")
+
+
+def _get_target_from_args(args: Dict[str, Any]) -> str:
+    """Extracts target string from args using common parameter keys."""
+    target = args.get("target") or args.get("query") or args.get("url") or args.get("target_resource") or args.get("claim") or ""
+    return str(target).strip()
+
+
+def _auto_log_epistemic_if_requested(args: Dict[str, Any], action_name: str, target: str, res: ResearchResult):
+    """
+    Optionally logs successful scraped result as a [HYPOTHESIS] epistemic item if requested.
+    Failed retrievals (res.ok == False) are strictly excluded from epistemic logging.
+    Scraped raw web data is tagged as [HYPOTHESIS] so agents are required to
+    cross-check and verify evidence before promoting claims to [PROVEN].
+    """
+    auto_log = args.get("auto_log_epistemic") or args.get("auto_log")
+    if not auto_log:
+        return
+
+    # Do not auto-log failed retrievals
+    if not res.ok:
+        logger.debug(f"Skipping auto_log_epistemic for {action_name}: retrieval failed.")
+        return
+
+    session_name = args.get("session_name", "").strip()
+    if not session_name:
+        if ACTIVE_SESSIONS:
+            session_name = next(iter(ACTIVE_SESSIONS.keys()))
+        else:
+            return
+    try:
+        session = get_or_load_session(session_name)
+        claim = f"Retrieved {action_name} research candidate: '{target}' ({res.title or 'Untitled'})"
+        evidence_content = res.to_markdown()
+        evidence_snippet = evidence_content[:1500] if len(evidence_content) > 1500 else evidence_content
+        # Log as HYPOTHESIS candidate evidence
+        session.log_epistemic_item("HYPOTHESIS", claim, evidence_snippet)
+    except Exception as e:
+        logger.warning(f"Could not auto-log epistemic hypothesis for {action_name}: {e}")
+
+
+def _handle_scrape_web(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'url' or 'query' for scrape_web."
+    res = scrape_web(target)
+    _auto_log_epistemic_if_requested(args, "scrape_web", target, res)
+    return res.to_markdown()
+
+
+def _handle_scrape_youtube(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'url' or 'video_id' for scrape_youtube."
+    res = scrape_youtube(target)
+    _auto_log_epistemic_if_requested(args, "scrape_youtube", target, res)
+    return res.to_markdown()
+
+
+def _handle_scrape_reddit(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'url' or 'subreddit' for scrape_reddit."
+    res = scrape_reddit(target)
+    _auto_log_epistemic_if_requested(args, "scrape_reddit", target, res)
+    return res.to_markdown()
+
+
+def _handle_scrape_x(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'url' or 'handle' for scrape_x."
+    res = scrape_x(target)
+    _auto_log_epistemic_if_requested(args, "scrape_x", target, res)
+    return res.to_markdown()
+
+
+def _handle_scrape_github(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'repo' or 'query' for scrape_github."
+    res = scrape_github(target)
+    _auto_log_epistemic_if_requested(args, "scrape_github", target, res)
+    return res.to_markdown()
+
+
+def _handle_scrape_arxiv(args: Dict[str, Any]) -> str:
+    target = _get_target_from_args(args)
+    if not target:
+        return "Error: Missing required parameter 'target' or 'paper_id' or 'query' for scrape_arxiv."
+    res = scrape_arxiv(target)
+    _auto_log_epistemic_if_requested(args, "scrape_arxiv", target, res)
+    return res.to_markdown()

fable_engine/schema.py

@@ -74,7 +74,13 @@
                     "generate_design_tokens",
                     "generate_awwwards_scaffold",
                     "validate_preflight_design",
-                    "list_design_archetypes"
+                    "list_design_archetypes",
+                    "scrape_web",
+                    "scrape_youtube",
+                    "scrape_reddit",
+                    "scrape_x",
+                    "scrape_github",
+                    "scrape_arxiv"
                 ],
                 "description": "The Fable session action to perform."
             },
@@ -525,6 +531,14 @@
             "user_prompt": {
                 "type": "string",
                 "description": "Alias for prompt: high-level design prompt describing desired web experience."
+            },
+            "target": {
+                "type": "string",
+                "description": "Target URL, ID, subreddit, repo, paper ID, handle, or search query string for research scraping actions."
+            },
+            "auto_log_epistemic": {
+                "type": "boolean",
+                "description": "Whether to automatically record retrieved research findings as a [HYPOTHESIS] candidate item in Fable Session's epistemic ledger for subsequent cross-verification."
             }
         },
         "required": ["action"]

fable_engine/scrapers/__init__.py

@@ -0,0 +1,32 @@
+"""
+Fable Engine Research Scrapers Package.
+Exporting zero-cost, standard-library based scrapers for Web, YouTube, Reddit, X, GitHub, and arXiv.
+"""
+
+from __future__ import annotations
+
+from fable_engine.scrapers.arxiv import ArxivScraper, scrape_arxiv
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url
+from fable_engine.scrapers.github import GitHubScraper, scrape_github
+from fable_engine.scrapers.reddit import RedditScraper, scrape_reddit
+from fable_engine.scrapers.web import WebScraper, scrape_web
+from fable_engine.scrapers.x import XScraper, scrape_x
+from fable_engine.scrapers.youtube import YouTubeScraper, scrape_youtube
+
+__all__ = [
+    "ResearchResult",
+    "ResearchSource",
+    "fetch_url",
+    "WebScraper",
+    "scrape_web",
+    "YouTubeScraper",
+    "scrape_youtube",
+    "RedditScraper",
+    "scrape_reddit",
+    "XScraper",
+    "scrape_x",
+    "GitHubScraper",
+    "scrape_github",
+    "ArxivScraper",
+    "scrape_arxiv",
+]

fable_engine/scrapers/arxiv.py

@@ -0,0 +1,115 @@
+"""
+arXiv research paper and search query scraper implementation using arXiv export API.
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+import xml.etree.ElementTree as ET
+from typing import Optional
+
+from fable_engine.scrapers.base import ResearchResult, ResearchSource, fetch_url
+
+
+class ArxivScraper(ResearchSource):
+    source_type = "arxiv"
+
+    def fetch(self, target: str, timeout: int = 15, max_content_length: int = 10000) -> ResearchResult:
+        target = target.strip()
+        if not target:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url="",
+                error="Target arXiv paper ID or search query cannot be empty."
+            )
+
+        paper_id = None
+        if "arxiv.org/" in target:
+            id_match = re.search(r"(?:abs|pdf)/([0-9]+\.[0-9]+(?:v[0-9]+)?)", target)
+            if id_match:
+                paper_id = id_match.group(1)
+        elif re.match(r"^[0-9]{4}\.[0-9]{4,5}(?:v[0-9]+)?$", target):
+            paper_id = target
+
+        if paper_id:
+            api_url = f"https://export.arxiv.org/api/query?id_list={paper_id}"
+            canonical_url = f"https://arxiv.org/abs/{paper_id}"
+        else:
+            query_enc = urllib.parse.quote(target)
+            api_url = f"https://export.arxiv.org/api/query?search_query=all:{query_enc}&max_results=5"
+            canonical_url = f"https://arxiv.org/search/?query={query_enc}&searchtype=all"
+
+        try:
+            xml_str = fetch_url(api_url, timeout=timeout)
+            root = ET.fromstring(xml_str)
+            ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
+
+            entries = root.findall("atom:entry", ns)
+            if not entries:
+                return ResearchResult(
+                    ok=True,
+                    source_type=self.source_type,
+                    canonical_url=canonical_url,
+                    title=f"arXiv Search: '{target}'",
+                    content="*(No papers found for this query)*"
+                )
+
+            blocks = []
+            first_title = ""
+            first_author = ""
+            for entry in entries:
+                title_elem = entry.find("atom:title", ns)
+                title = re.sub(r"\s+", " ", title_elem.text).strip() if title_elem is not None and title_elem.text else "Untitled"
+                if not first_title:
+                    first_title = title
+
+                id_elem = entry.find("atom:id", ns)
+                entry_id = id_elem.text.strip() if id_elem is not None and id_elem.text else ""
+
+                published_elem = entry.find("atom:published", ns)
+                published = published_elem.text[:10] if published_elem is not None and published_elem.text else ""
+
+                summary_elem = entry.find("atom:summary", ns)
+                summary = re.sub(r"\s+", " ", summary_elem.text).strip() if summary_elem is not None and summary_elem.text else ""
+
+                authors = []
+                for author_elem in entry.findall("atom:author", ns):
+                    name_elem = author_elem.find("atom:name", ns)
+                    if name_elem is not None and name_elem.text:
+                        authors.append(name_elem.text.strip())
+                if not first_author and authors:
+                    first_author = ", ".join(authors)
+
+                pdf_link = entry_id.replace("/abs/", "/pdf/") + ".pdf" if "/abs/" in entry_id else entry_id
+
+                blocks.append(f"## {title}")
+                blocks.append(f"**Authors**: {', '.join(authors)}")
+                blocks.append(f"**Published**: {published} | **arXiv**: [{entry_id}]({entry_id}) | **PDF**: [{pdf_link}]({pdf_link})\n")
+                blocks.append(f"### Abstract\n{summary}\n")
+
+            full_content = "\n".join(blocks)
+            if len(full_content) > max_content_length:
+                full_content = full_content[:max_content_length] + "\n\n*(Abstracts truncated)*"
+
+            return ResearchResult(
+                ok=True,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                title=first_title if paper_id else f"arXiv Papers for: '{target}'",
+                author=first_author,
+                content=full_content,
+                metadata={"paper_count": len(entries)}
+            )
+        except Exception as e:
+            return ResearchResult(
+                ok=False,
+                source_type=self.source_type,
+                canonical_url=canonical_url,
+                error=f"arXiv fetch failed: {str(e)}"
+            )
+
+
+def scrape_arxiv(target: str, timeout: int = 15) -> ResearchResult:
+    return ArxivScraper().fetch(target, timeout=timeout)