From 82ebd2ad98ec1fe271c5f9ea7bece6d4d5210083 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Fri, 13 Feb 2026 09:26:06 -0800 Subject: [PATCH 01/26] Add OG meta tags to demo and dashboard pages Add Open Graph and Twitter Card meta tags so sharing links on Slack, Discord, Twitter, etc. shows a proper preview card with the OSA branding image from osc.earth. --- dashboard/osa/index.html | 15 +++++++++++++++ frontend/index.html | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/dashboard/osa/index.html b/dashboard/osa/index.html index c3b435d..17a32f1 100644 --- a/dashboard/osa/index.html +++ b/dashboard/osa/index.html @@ -4,6 +4,21 @@ Open Science Assistant - Dashboard + + + + + + + + + + + + + + + -
+
+
+ +
+
+
@@ -442,5 +525,10 @@

Learn More

// Load communities from API and then route loadCommunities(); + +
+ © + with ❤️ OpenScience Collective +
From b68673677b641c81ab727617e35dfe96bf97038a Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Fri, 13 Feb 2026 10:20:00 -0800 Subject: [PATCH 03/26] Reorder footer: copyright holder first, then made with love --- dashboard/osa/index.html | 3 ++- frontend/index.html | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dashboard/osa/index.html b/dashboard/osa/index.html index 5dda99c..93cf92f 100644 --- a/dashboard/osa/index.html +++ b/dashboard/osa/index.html @@ -459,7 +459,8 @@

Admin Access

© - with ❤️ OpenScience Collective + OpenScience Collective + · Made with ❤️ · Documentation · diff --git a/frontend/index.html b/frontend/index.html index cccf5ea..316c10a 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -528,7 +528,8 @@

Learn More

© - with ❤️ OpenScience Collective + OpenScience Collective + · Made with ❤️
From d86d7e9bd91fe6b8ed6b1b635c871e3f590f28a4 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Fri, 13 Feb 2026 13:47:06 -0800 Subject: [PATCH 04/26] Fix review issues: dark mode, meta tags, broken links - Add dark mode to dashboard (matching frontend theme) - Add to both pages for SEO - Remove redundant twitter:title/description/image tags - Fix broken GitHub links (osc-em -> OpenScience-Collective) - Standardize org name to "OpenScience Collective" - Move dashboard inline styles to CSS classes - Rename .footer to .site-footer for consistency --- dashboard/osa/index.html | 82 ++++++++++++++++++++++++++++++++++------ frontend/index.html | 9 ++--- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/dashboard/osa/index.html b/dashboard/osa/index.html index 93cf92f..ef1572e 100644 --- a/dashboard/osa/index.html +++ b/dashboard/osa/index.html @@ -15,9 +15,8 @@ - - - + +
-
-
-
+
© OpenScience Collective · Made with ❤️ · Documentation · - Source + Source
") + + def test_logo_url_rejects_ftp(self) -> None: + """Should reject ftp: URLs.""" + with pytest.raises(ValidationError, match="logo_url must use"): + WidgetConfig(logo_url="ftp://example.com/logo.png") + + def test_logo_url_none_by_default(self) -> None: + """Should default to None.""" + widget = WidgetConfig() + assert widget.logo_url is None + + def test_logo_url_empty_string_normalized(self) -> None: + """Empty or whitespace-only string should become None.""" + widget = WidgetConfig(logo_url=" ") + assert widget.logo_url is None + + def test_logo_url_strips_whitespace(self) -> None: + """Should strip whitespace from logo_url.""" + widget = WidgetConfig(logo_url=" https://example.com/logo.png ") + assert widget.logo_url == "https://example.com/logo.png" + + def test_resolve_with_logo_url_fallback(self) -> None: + """resolve() should use fallback logo_url when self.logo_url is None.""" + widget = WidgetConfig() + result = widget.resolve("Test", logo_url="/test/logo") + assert result["logo_url"] == "/test/logo" + + def test_resolve_explicit_logo_url_takes_precedence(self) -> None: + """resolve() should prefer explicit logo_url over fallback.""" + widget = WidgetConfig(logo_url="https://example.com/explicit.png") + result = widget.resolve("Test", logo_url="/test/logo") + assert result["logo_url"] == "https://example.com/explicit.png" + + def test_resolve_no_logo_url_returns_none(self) -> None: + """resolve() should return None when no logo_url is set anywhere.""" + widget = WidgetConfig() + result = widget.resolve("Test") + assert result["logo_url"] is None + + class TestCommunityConfigWidget: """Tests for CommunityConfig.widget field.""" From c229fd1054ae2383ea5aab6ce9cc8050dc157a9a Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Tue, 24 Feb 2026 20:56:52 -0800 Subject: [PATCH 22/26] fix: add theme_color to WidgetConfigResponse --- src/api/routers/community.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/routers/community.py b/src/api/routers/community.py index 1022cc3..a8c4474 100644 --- a/src/api/routers/community.py +++ b/src/api/routers/community.py @@ -178,6 +178,7 @@ class WidgetConfigResponse(BaseModel): logo_url: str | None = Field( default=None, description="URL for community logo/icon in widget header" ) + theme_color: str | None = Field(default=None, description="Primary theme color as hex #RRGGBB") class CommunityConfigResponse(BaseModel): From ee6b79395b89536bddfaae933e199230140fd33f Mon Sep 17 00:00:00 2001 From: "Seyed (Yahya) Shirazi" Date: Wed, 25 Feb 2026 04:12:19 -0800 Subject: [PATCH 23/26] feat: onboard MNE-Python community with Discourse syncer (#230) * feat: onboard MNE-Python community with Discourse syncer Add MNE-Python as a new community with 31 documentation pages, 5 GitHub repos, 5 citation DOIs, Python docstring extraction, and Discourse forum sync. Build a generic Discourse syncer using the public JSON API with patient rate limiting (1 req/s). Key changes: - MNE community config (config.yaml) with full ecosystem - Generic Discourse syncer (discourse_sync.py) with incremental sync - DB schema: discourse_topics table with FTS5 search - HTML-to-markdown auto-conversion in doc fetcher (for Sphinx docs) - Discourse search tool factory and CLI sync command - DiscourseCategoryConfig and SyncConfig.discourse schema additions - Tests for DB, search, config validation, and live API Closes #229 * fix: address PR review findings for Discourse syncer - Wrap per-topic sync loop in try/except to prevent single bad topic from crashing the entire sync - Use .get() for topic id/slug to avoid KeyError on malformed API responses - Add logging when listing pagination fails - Move get_last_sync to top-level import - Include discourse count in sync_all total_items summary - Remove redundant snippet truncation in discourse tool * Add missing tests for discourse tools and helpers - TestSearchDiscourseTool: missing table, no results, correct naming - TestHtmlToText: simple HTML, empty input, code blocks, whitespace - TestGetAcceptedAnswer: accepted answer, fallback to liked, no replies - Updated test_includes_all_optional_tools to include discourse * Fix comment accuracy issues from review - Rename _html_to_text to _html_to_markdown (matches actual behavior) - Fix _get_accepted_answer docstring: "most-liked" not "highest-scoring" - Update get_last_sync/update_sync_metadata docs to include 'discourse' * Improve type safety for Discourse category config - Use DiscourseCategoryConfig directly instead of list[dict] - Remove model_dump() type laundering in sync.py - Add field validation: slug pattern, id >= 1 - Change accepted_answer_snippet to str | None --- src/assistants/community.py | 2 + src/assistants/mne/config.yaml | 471 ++++++++++++++++++++ src/cli/sync.py | 72 ++- src/core/config/community.py | 18 + src/knowledge/__init__.py | 4 + src/knowledge/db.py | 149 ++++++- src/knowledge/discourse_sync.py | 384 ++++++++++++++++ src/knowledge/search.py | 86 ++++ src/tools/fetcher.py | 72 ++- src/tools/knowledge.py | 84 ++++ tests/test_knowledge/test_db.py | 2 +- tests/test_knowledge/test_discourse_sync.py | 371 +++++++++++++++ tests/test_tools/test_knowledge_tools.py | 55 ++- 13 files changed, 1761 insertions(+), 9 deletions(-) create mode 100644 src/assistants/mne/config.yaml create mode 100644 src/knowledge/discourse_sync.py create mode 100644 tests/test_knowledge/test_discourse_sync.py diff --git a/src/assistants/community.py b/src/assistants/community.py index 9288ef0..c65860a 100644 --- a/src/assistants/community.py +++ b/src/assistants/community.py @@ -228,6 +228,7 @@ def _build_tools(self, config: CommunityConfig) -> list[BaseTool]: has_docstrings = config.docstrings and config.docstrings.repos has_faq = config.faq_generation is not None and bool(config.mailman) + has_discourse = bool(config.discourse) knowledge_tools = create_knowledge_tools( community_id=config.id, @@ -239,6 +240,7 @@ def _build_tools(self, config: CommunityConfig) -> list[BaseTool]: include_docstrings=bool(has_docstrings), include_faq=bool(has_faq), faq_list_names=([m.list_name for m in config.mailman] if config.mailman else None), + include_discourse=bool(has_discourse), ) tools.extend(knowledge_tools) diff --git a/src/assistants/mne/config.yaml b/src/assistants/mne/config.yaml new file mode 100644 index 0000000..b2aa590 --- /dev/null +++ b/src/assistants/mne/config.yaml @@ -0,0 +1,471 @@ +# MNE-Python Assistant Configuration +# Single source of truth for the MNE community assistant + +id: mne +name: MNE-Python +description: Open-source Python toolkit for exploring, visualizing, and analyzing human neurophysiological data (MEG, EEG, sEEG, ECoG, and NIRS) +status: available +default_model: anthropic/claude-haiku-4.5 +default_model_provider: anthropic + +# External links for dashboard and discovery +links: + homepage: https://mne.tools + documentation: https://mne.tools/stable/ + repository: https://github.com/mne-tools + demo: https://demo.osc.earth/mne + +# Widget configuration for frontend embedding +widget: + title: MNE-Python Assistant + theme_color: "#1f77b4" + initial_message: "Hi! I'm the MNE-Python Assistant. I can help with MEG, EEG, and neurophysiological data analysis using MNE-Python and its ecosystem." + placeholder: Ask about MNE-Python... + suggested_questions: + - How do I read raw EEG data in MNE? + - How do I filter and preprocess my data? + - How do I run ICA for artifact removal? + - How do I perform source localization? + - How do I do time-frequency analysis? + +cors_origins: + - https://mne.tools + - https://*.mne.tools + +# Budget limits for cost management +budget: + daily_limit_usd: 5.00 + monthly_limit_usd: 50.00 + alert_threshold_pct: 80.0 + +# System prompt template with runtime-substituted placeholders +system_prompt: | + You are a technical assistant specialized in helping users with MNE-Python, an open-source Python toolkit for exploring, visualizing, and analyzing human neurophysiological data including MEG, EEG, sEEG, ECoG, and NIRS. + The MNE ecosystem includes MNE-Python (core library), MNE-BIDS (BIDS format support), MNE-Connectivity (spectral and effective connectivity), MNE-ICALabel (automatic ICA component labeling), and MNE-LSL (real-time data streaming). + You provide explanations, troubleshooting, and step-by-step guidance for neurophysiological data analysis workflows in Python. + Focus on helping users with MNE-Python and MEG/EEG/NIRS analysis. You may reference related concepts (signal processing, BIDS, source modeling theory, machine learning) when they help answer the user's question. + Base your responses on official MNE documentation, established best practices, and the tools available to you. + Always attempt to answer the user's question. Use the documentation and search tools to look up information + you're unsure about rather than declining to answer. If specific details aren't available in the docs, + provide what you do know and note which parts you're less certain about. + + When a user's question is ambiguous, assume the most likely meaning and provide a useful starting point, + but also ask clarifying questions when necessary. + Communicate in a clear and technical style, prioritizing accuracy while remaining accessible. + Balance clarity and technical precision, starting with practical guidance and expanding into details when needed. + Answers should be well-structured and easy to follow, with examples and code snippets where appropriate. + + The MNE homepage is https://mne.tools + MNE tutorials and documentation are at https://mne.tools/stable/ + The MNE GitHub organization is at https://github.com/mne-tools + The MNE community forum is at https://mne.discourse.group/ + MNE API reference is at https://mne.tools/stable/api/python_reference.html + + ## Response Style (IMPORTANT -- follow this strictly) + + Your responses must be structured but brief: + - Use markdown headers to organize, but keep each section to 1-2 sentences + - Aim for 200-300 words total. Never exceed 400 words unless the user asks for detail. + - End with 2-3 specific follow-up suggestions so the user drives the conversation + - Do NOT give exhaustive answers on the first response. This is a conversation, not a lecture. + - When showing code examples, show ONE focused snippet, not complete workflows + + ## MNE-Python Data Pipeline + + MNE-Python follows a structured data pipeline. Guide users through the appropriate stage: + + **Core data structures (in analysis order):** + 1. `Raw` - Continuous time series data (loading, filtering, preprocessing) + 2. `Epochs` - Time-locked segments around events + 3. `Evoked` - Averaged epoch data (ERPs/ERFs) + 4. `SourceEstimate` - Source-level activity estimates + + **Common workflow:** + ```python + import mne + + # Load data + raw = mne.io.read_raw_fif('data_raw.fif', preload=True) + + # Preprocess + raw.filter(l_freq=1.0, h_freq=40.0) + + # Create epochs + events = mne.find_events(raw) + epochs = mne.Epochs(raw, events, tmin=-0.2, tmax=0.5) + + # Average + evoked = epochs.average() + ``` + + ## Using Tools Strategically + + You have access to tools for documentation retrieval and knowledge discovery. Use them to verify facts and ensure accuracy. + + - Retrieve documentation when you need specific information not covered by preloaded docs + - When users ask about recent activity, issues, or PRs, use the knowledge discovery tools + - When users ask about research papers, use the paper search tool + - When users ask about function parameters or usage, use the docstring search tool + - When users ask about common problems, use the forum search tool + + ## Using the retrieve_mne_docs Tool + + Retrieve documentation when you need to verify specifics or the user asks about a topic not covered by preloaded docs. + Include links to relevant documents when you cite them. + + **Important guidelines:** + - Do NOT retrieve docs that have already been preloaded (listed below) + - Use preloaded docs first; only fetch additional docs when needed + - If you have already loaded a document in this conversation, don't load it again + + {preloaded_docs_section} + + {available_docs_section} + + ## Available Tools + + You have access to multiple specialized tools to help researchers: + + **Documentation & Codebase:** + 1. `retrieve_mne_docs`: Fetch tutorials and guides from mne.tools + 2. `search_mne_code_docs`: Search Python function/class documentation from MNE codebase + + **Community Knowledge:** + 3. `search_mne_discussions`: Search GitHub issues and PRs across MNE repos + 4. `list_mne_recent`: List recent development activity (PRs, issues) + 5. `search_mne_forum`: Search MNE Discourse forum topics and answers + + **Research:** + 6. `search_mne_papers`: Search academic literature about MNE and neurophysiology + + ## Tool Usage Guidelines + + **For function usage questions:** Use `search_mne_code_docs` first + - Example: "How do I use read_raw_edf?" -> CALL `search_mne_code_docs(query="read_raw_edf")` + - Example: "What parameters does Epochs accept?" -> CALL `search_mne_code_docs(query="Epochs")` + + **For common problems and troubleshooting:** Use `search_mne_forum` to find past solutions + - Example: "How to fix rank deficiency in ICA?" -> CALL `search_mne_forum(query="rank deficiency ICA")` + - Example: "Error with forward model?" -> CALL `search_mne_forum(query="forward model error")` + + **For current development issues:** Use `search_mne_discussions` + - Example: "Any open issues with NIRS?" -> CALL `search_mne_discussions(query="NIRS", status="open")` + + **For tutorials and guides:** Use `retrieve_mne_docs` + - Example: "Show me the ICA tutorial" -> CALL `retrieve_mne_docs(title="ICA")` + + Always cite sources with links to documentation, GitHub issues, or forum threads. + + ## Knowledge Discovery Tools - YOU MUST USE THESE + + You have access to a synced knowledge database with GitHub issues, PRs, academic papers, function documentation, and Discourse forum Q&A. + **You MUST use these tools when users ask about recent activity, issues, PRs, function usage, or troubleshooting.** + + **Available MNE repositories in the database:** + {repo_list} + + **CRITICAL: When users mention these repos (even by short name), USE THE TOOLS:** + - "mne" or "mne-python" -> repo="mne-tools/mne-python" + - "mne-bids" or "bids" -> repo="mne-tools/mne-bids" + - "connectivity" -> repo="mne-tools/mne-connectivity" + - "icalabel" -> repo="mne-tools/mne-icalabel" + - "lsl" or "mne-lsl" -> repo="mne-tools/mne-lsl" + + **MANDATORY: Use tools for these question patterns:** + - "What are the latest PRs?" -> CALL `list_mne_recent(item_type="pr")` + - "Latest PRs in mne-bids?" -> CALL `list_mne_recent(item_type="pr", repo="mne-tools/mne-bids")` + - "Open issues?" -> CALL `list_mne_recent(item_type="issue", status="open")` + - "Any discussions about source localization?" -> CALL `search_mne_discussions(query="source localization")` + + **Core MNE papers tracked for citations (DOIs in database):** + {paper_dois} + + **MANDATORY: Use tools for citation/paper questions:** + - "Papers about MNE?" -> CALL `search_mne_papers(query="MNE-Python")` + - "Research on connectivity analysis?" -> CALL `search_mne_papers(query="connectivity MEG EEG")` + + **DO NOT:** + - Tell users to "visit GitHub", "check Google Scholar", or "use the API" when you have the data + - Make up PR numbers, issue numbers, paper titles, authors, or citation counts + - Say "I don't have access" - you DO have access via the tools above + - Hallucinate fake papers, fake authors, or fake citation counts + + **Present results as discovery:** + - "Here are the recent PRs in MNE-Python: [actual list with real URLs]" + - "There's a related discussion: [real link]" + - "Here are papers related to MNE: [actual list from database with real URLs]" + + The knowledge database may not be populated. If you get a message about initializing the database, + then explain that the knowledge base isn't set up yet. + + {page_context_section} + + {additional_instructions} + +# Documentation sources +# - preload: true = embedded in system prompt (recommended: 2-3 core docs to keep prompt lean) +# - preload: false/omitted = fetched on demand via retrieve_docs tool +# Note: MNE docs are Sphinx-generated HTML. The fetcher auto-converts HTML to markdown. +documentation: + # === PRELOADED: Core overview (2 docs) === + - title: The typical M/EEG workflow + url: https://mne.tools/stable/documentation/cookbook.html + source_url: https://mne.tools/stable/documentation/cookbook.html + preload: true + category: core + description: Overview of the standard MEG/EEG analysis workflow with MNE-Python. + + - title: The MNE tools suite + url: https://mne.tools/stable/install/mne_tools_suite.html + source_url: https://mne.tools/stable/install/mne_tools_suite.html + preload: true + category: core + description: Overview of MNE-Python and related tools in the MNE ecosystem. + + # === ON-DEMAND: Introduction (3 docs) === + - title: Overview of MEG/EEG analysis with MNE-Python + url: https://mne.tools/stable/auto_tutorials/intro/10_overview.html + source_url: https://mne.tools/stable/auto_tutorials/intro/10_overview.html + category: intro + description: Getting started with MNE-Python, basic concepts and data loading. + + - title: The Info data structure + url: https://mne.tools/stable/auto_tutorials/intro/30_info.html + source_url: https://mne.tools/stable/auto_tutorials/intro/30_info.html + category: intro + description: Understanding MNE's Info object for measurement metadata. + + - title: Working with sensor locations + url: https://mne.tools/stable/auto_tutorials/intro/40_sensor_locations.html + source_url: https://mne.tools/stable/auto_tutorials/intro/40_sensor_locations.html + category: intro + description: Loading, plotting, and managing sensor/electrode positions. + + # === ON-DEMAND: Data I/O (3 docs) === + - title: Importing data from MEG devices + url: https://mne.tools/stable/auto_tutorials/io/10_reading_meg_data.html + source_url: https://mne.tools/stable/auto_tutorials/io/10_reading_meg_data.html + category: io + description: Reading data from Elekta, CTF, BTi, KIT, and other MEG systems. + + - title: Importing data from EEG devices + url: https://mne.tools/stable/auto_tutorials/io/20_reading_eeg_data.html + source_url: https://mne.tools/stable/auto_tutorials/io/20_reading_eeg_data.html + category: io + description: Reading EDF, BDF, EGI, BrainVision, EEGLAB, and other EEG formats. + + - title: Importing data from fNIRS devices + url: https://mne.tools/stable/auto_tutorials/io/30_reading_fnirs_data.html + source_url: https://mne.tools/stable/auto_tutorials/io/30_reading_fnirs_data.html + category: io + description: Reading functional near-infrared spectroscopy data. + + # === ON-DEMAND: Raw data (2 docs) === + - title: The Raw data structure + url: https://mne.tools/stable/auto_tutorials/raw/10_raw_overview.html + source_url: https://mne.tools/stable/auto_tutorials/raw/10_raw_overview.html + category: raw + description: Working with continuous data, the Raw object, and basic operations. + + - title: Annotating continuous data + url: https://mne.tools/stable/auto_tutorials/raw/30_annotate_raw.html + source_url: https://mne.tools/stable/auto_tutorials/raw/30_annotate_raw.html + category: raw + description: Adding annotations to mark bad segments, events, and artifacts. + + # === ON-DEMAND: Preprocessing (5 docs) === + - title: Overview of artifact detection + url: https://mne.tools/stable/auto_tutorials/preprocessing/10_preprocessing_overview.html + source_url: https://mne.tools/stable/auto_tutorials/preprocessing/10_preprocessing_overview.html + category: preprocessing + description: Overview of preprocessing steps and artifact detection strategies. + + - title: Filtering and resampling data + url: https://mne.tools/stable/auto_tutorials/preprocessing/30_filtering_resampling.html + source_url: https://mne.tools/stable/auto_tutorials/preprocessing/30_filtering_resampling.html + category: preprocessing + description: Applying FIR and IIR filters, downsampling, and anti-aliasing. + + - title: Repairing artifacts with ICA + url: https://mne.tools/stable/auto_tutorials/preprocessing/40_artifact_correction_ica.html + source_url: https://mne.tools/stable/auto_tutorials/preprocessing/40_artifact_correction_ica.html + category: preprocessing + description: Using Independent Component Analysis to remove eye blinks, heartbeat, and other artifacts. + + - title: Setting the EEG reference + url: https://mne.tools/stable/auto_tutorials/preprocessing/55_setting_eeg_reference.html + source_url: https://mne.tools/stable/auto_tutorials/preprocessing/55_setting_eeg_reference.html + category: preprocessing + description: Re-referencing EEG data to average, REST, or specific electrode references. + + - title: Signal-space separation (SSS) and Maxwell filtering + url: https://mne.tools/stable/auto_tutorials/preprocessing/60_maxwell_filtering_sss.html + source_url: https://mne.tools/stable/auto_tutorials/preprocessing/60_maxwell_filtering_sss.html + category: preprocessing + description: MEG-specific noise reduction using Maxwell filtering and SSS. + + # === ON-DEMAND: Epochs (3 docs) === + - title: The Epochs data structure + url: https://mne.tools/stable/auto_tutorials/epochs/10_epochs_overview.html + source_url: https://mne.tools/stable/auto_tutorials/epochs/10_epochs_overview.html + category: epochs + description: Creating, manipulating, and understanding epoched data. + + - title: Working with Epoch metadata + url: https://mne.tools/stable/auto_tutorials/epochs/30_epochs_metadata.html + source_url: https://mne.tools/stable/auto_tutorials/epochs/30_epochs_metadata.html + category: epochs + description: Using pandas DataFrames as metadata for advanced epoch selection. + + - title: Visualizing epoched data + url: https://mne.tools/stable/auto_tutorials/epochs/20_visualize_epochs.html + source_url: https://mne.tools/stable/auto_tutorials/epochs/20_visualize_epochs.html + category: epochs + description: Plotting epochs, image plots, and drop logs. + + # === ON-DEMAND: Evoked responses (2 docs) === + - title: The Evoked data structure + url: https://mne.tools/stable/auto_tutorials/evoked/10_evoked_overview.html + source_url: https://mne.tools/stable/auto_tutorials/evoked/10_evoked_overview.html + category: evoked + description: Working with averaged evoked data (ERPs and ERFs). + + - title: EEG analysis - Event-Related Potentials (ERPs) + url: https://mne.tools/stable/auto_tutorials/evoked/30_eeg_erp.html + source_url: https://mne.tools/stable/auto_tutorials/evoked/30_eeg_erp.html + category: evoked + description: Complete ERP analysis workflow from raw data to group statistics. + + # === ON-DEMAND: Time-frequency (2 docs) === + - title: The Spectrum and EpochsSpectrum classes + url: https://mne.tools/stable/auto_tutorials/time-freq/10_spectrum_class.html + source_url: https://mne.tools/stable/auto_tutorials/time-freq/10_spectrum_class.html + category: time_freq + description: Computing and visualizing power spectra with Welch and multitaper methods. + + - title: Frequency and time-frequency sensor analysis + url: https://mne.tools/stable/auto_tutorials/time-freq/20_sensors_time_frequency.html + source_url: https://mne.tools/stable/auto_tutorials/time-freq/20_sensors_time_frequency.html + category: time_freq + description: Morlet wavelets, ERSP, inter-trial coherence, and induced power. + + # === ON-DEMAND: Forward modeling (2 docs) === + - title: Source alignment and coordinate frames + url: https://mne.tools/stable/auto_tutorials/forward/20_source_alignment.html + source_url: https://mne.tools/stable/auto_tutorials/forward/20_source_alignment.html + category: forward + description: Coregistration of MRI and MEG/EEG coordinate systems. + + - title: Head model and forward computation + url: https://mne.tools/stable/auto_tutorials/forward/30_forward.html + source_url: https://mne.tools/stable/auto_tutorials/forward/30_forward.html + category: forward + description: Computing BEM surfaces, source spaces, and forward solutions. + + # === ON-DEMAND: Source localization (3 docs) === + - title: Source localization with MNE, dSPM, sLORETA, and eLORETA + url: https://mne.tools/stable/auto_tutorials/inverse/30_mne_dspm_loreta.html + source_url: https://mne.tools/stable/auto_tutorials/inverse/30_mne_dspm_loreta.html + category: inverse + description: Distributed source estimation using minimum-norm methods. + + - title: Source reconstruction using an LCMV beamformer + url: https://mne.tools/stable/auto_tutorials/inverse/50_beamformer_lcmv.html + source_url: https://mne.tools/stable/auto_tutorials/inverse/50_beamformer_lcmv.html + category: inverse + description: Beamforming for source localization using LCMV spatial filter. + + - title: Visualize source time courses (stcs) + url: https://mne.tools/stable/auto_tutorials/inverse/60_visualize_stc.html + source_url: https://mne.tools/stable/auto_tutorials/inverse/60_visualize_stc.html + category: inverse + description: Plotting source estimates on brain surfaces and volumes. + + # === ON-DEMAND: Statistics (2 docs) === + - title: Statistical inference + url: https://mne.tools/stable/auto_tutorials/stats-sensor-space/10_background_stats.html + source_url: https://mne.tools/stable/auto_tutorials/stats-sensor-space/10_background_stats.html + category: stats + description: Overview of parametric and non-parametric statistical tests in MNE. + + - title: Non-parametric cluster permutation statistics + url: https://mne.tools/stable/auto_tutorials/stats-sensor-space/40_cluster_1samp_time_freq.html + source_url: https://mne.tools/stable/auto_tutorials/stats-sensor-space/40_cluster_1samp_time_freq.html + category: stats + description: Cluster-based permutation tests for time-frequency data. + + # === ON-DEMAND: Machine learning (1 doc) === + - title: Decoding (MVPA) + url: https://mne.tools/stable/auto_tutorials/machine-learning/50_decoding.html + source_url: https://mne.tools/stable/auto_tutorials/machine-learning/50_decoding.html + category: machine_learning + description: Multi-variate pattern analysis and temporal generalization. + + # === ON-DEMAND: Clinical (1 doc) === + - title: Working with sEEG data + url: https://mne.tools/stable/auto_tutorials/clinical/20_seeg.html + source_url: https://mne.tools/stable/auto_tutorials/clinical/20_seeg.html + category: clinical + description: Analysis of stereo-EEG recordings with depth electrodes. + +# Sync schedule configuration +# Each sync type runs on its own cron schedule (UTC) +# Staggered to avoid concurrent load with other communities +sync: + github: + cron: "30 2 * * *" # daily at 2:30am UTC + papers: + cron: "30 3 * * 0" # weekly Sunday at 3:30am UTC + docstrings: + cron: "30 4 * * 1" # weekly Monday at 4:30am UTC + discourse: + cron: "30 5 * * 1" # weekly Monday at 5:30am UTC + +# GitHub repositories for issue/PR sync +github: + repos: + - mne-tools/mne-python + - mne-tools/mne-bids + - mne-tools/mne-connectivity + - mne-tools/mne-icalabel + - mne-tools/mne-lsl + +# Paper/citation search configuration +citations: + queries: + - MNE-Python + - MNE MEG EEG analysis + - MNE source localization + - MNE-BIDS + - MNE connectivity analysis + dois: + - "10.3389/fnins.2013.00267" # MEG and EEG Data Analysis with MNE-Python (Gramfort et al., 2013) + - "10.1016/j.neuroimage.2013.10.027" # MNE Software for Processing MEG and EEG Data (Gramfort et al., 2014) + - "10.21105/joss.01896" # MNE-BIDS (Appelhoff et al., 2019) + - "10.21105/joss.04484" # MNE-ICALabel (Li et al., 2022) + - "10.21105/joss.08088" # MNE-LSL (Scheltienne et al.) + +# Discourse forum configuration +discourse: + - url: https://mne.discourse.group + tags: [] + +# Docstring extraction configuration +# MNE ecosystem uses NumPy-style docstrings; our Python AST parser handles them +docstrings: + repos: + - repo: mne-tools/mne-python + branch: main + languages: [python] + - repo: mne-tools/mne-bids + branch: main + languages: [python] + - repo: mne-tools/mne-connectivity + branch: main + languages: [python] + - repo: mne-tools/mne-icalabel + branch: main + languages: [python] + - repo: mne-tools/mne-lsl + branch: main + languages: [python] diff --git a/src/cli/sync.py b/src/cli/sync.py index 9836222..87441fd 100644 --- a/src/cli/sync.py +++ b/src/cli/sync.py @@ -532,6 +532,7 @@ def sync_all( grand_github_total = 0 grand_paper_total = 0 grand_bep_total = 0 + grand_discourse_total = 0 for comm_id in communities: console.print(f"\n[bold cyan]═══ Syncing {comm_id} ═══[/bold cyan]") @@ -597,7 +598,28 @@ def sync_all( console.print(f"[red]BEP sync failed: {e}[/red]") logger.exception("BEP sync failed for %s", comm_id) - total_items = grand_github_total + grand_paper_total + grand_bep_total + # Discourse forum topics + comm_info = registry.get(comm_id) + if comm_info and comm_info.community_config and comm_info.community_config.discourse: + console.print("[bold]Syncing Discourse topics...[/bold]") + try: + from src.knowledge.discourse_sync import sync_discourse_topics + + discourse_total = 0 + for discourse_cfg in comm_info.community_config.discourse: + discourse_total += sync_discourse_topics( + base_url=str(discourse_cfg.url), + project=comm_id, + categories=discourse_cfg.categories or None, + incremental=not full, + ) + console.print(f"[green]Discourse: {discourse_total} topics[/green]") + grand_discourse_total += discourse_total + except Exception as e: + console.print(f"[red]Discourse sync failed: {e}[/red]") + logger.exception("Discourse sync failed for %s", comm_id) + + total_items = grand_github_total + grand_paper_total + grand_bep_total + grand_discourse_total community_word = "community" if len(communities) == 1 else "communities" console.print( f"\n[bold green]Sync complete: {total_items} total items " @@ -861,3 +883,51 @@ def sync_faq( table.add_row("Estimated cost", f"${result['total_cost']:.2f}") console.print(table) + + +@sync_app.command("discourse") +def sync_discourse( + community: Annotated[ + str, + typer.Option("--community", "-c", help="Community ID to sync (e.g., mne)"), + ] = "mne", + full: Annotated[ + bool, + typer.Option("--full", help="Full sync (not incremental)"), + ] = False, + max_topics: Annotated[ + int | None, + typer.Option("--max", help="Maximum topics to sync (for testing)"), + ] = None, +) -> None: + """Sync Discourse forum topics from a community's Discourse instance. + + Fetches topics and their posts from the Discourse public JSON API. + Stores topics with first post and best answer for search. + """ + _require_admin() + _validate_community(community) + + if not _safe_init_db(community): + raise typer.Exit(1) + + # Get discourse config from community + info = registry.get(community) + if not info or not info.community_config.discourse: + console.print(f"[red]Error: No Discourse forum configured for {community}[/red]") + raise typer.Exit(1) + + from src.knowledge.discourse_sync import sync_discourse_topics + + total = 0 + for discourse_config in info.community_config.discourse: + count = sync_discourse_topics( + base_url=str(discourse_config.url), + project=community, + categories=discourse_config.categories or None, + incremental=not full, + max_topics=max_topics, + ) + total += count + + console.print(f"\n[green]Synced {total} Discourse topics for {community}[/green]") diff --git a/src/core/config/community.py b/src/core/config/community.py index e385c65..a3ab38f 100644 --- a/src/core/config/community.py +++ b/src/core/config/community.py @@ -267,6 +267,18 @@ def validate_dois(cls, v: list[str]) -> list[str]: return list(dict.fromkeys(normalized)) +class DiscourseCategoryConfig(BaseModel): + """A Discourse category to sync.""" + + model_config = ConfigDict(extra="forbid") + + slug: str = Field(min_length=1, pattern=r"^[a-z0-9-]+$") + """Category slug (e.g., 'support').""" + + id: int = Field(ge=1) + """Category numeric ID.""" + + class DiscourseConfig(BaseModel): """Discourse/forum search configuration.""" @@ -278,6 +290,9 @@ class DiscourseConfig(BaseModel): tags: list[str] = Field(default_factory=list) """Tags to filter forum topics by.""" + categories: list[DiscourseCategoryConfig] = Field(default_factory=list) + """Optional categories to limit sync to. Empty means sync all.""" + class MailmanConfig(BaseModel): """Mailing list configuration for FAQ generation.""" @@ -814,6 +829,9 @@ class SyncConfig(BaseModel): beps: SyncTypeSchedule | None = None """Schedule for BIDS Extension Proposals sync (BIDS-specific).""" + discourse: SyncTypeSchedule | None = None + """Schedule for Discourse forum topic sync.""" + class CommunityConfig(BaseModel): """Configuration for a single research community assistant. diff --git a/src/knowledge/__init__.py b/src/knowledge/__init__.py index ee3f0cc..d41bdc6 100644 --- a/src/knowledge/__init__.py +++ b/src/knowledge/__init__.py @@ -16,18 +16,22 @@ from src.knowledge.db import get_connection, get_db_path, init_db from src.knowledge.search import ( BEPResult, + DiscourseTopicResult, SearchResult, search_beps, + search_discourse_topics, search_github_items, search_papers, ) __all__ = [ "BEPResult", + "DiscourseTopicResult", "get_connection", "get_db_path", "init_db", "search_beps", + "search_discourse_topics", "search_github_items", "search_papers", "SearchResult", diff --git a/src/knowledge/db.py b/src/knowledge/db.py index 7f49483..8ba2d77 100644 --- a/src/knowledge/db.py +++ b/src/knowledge/db.py @@ -304,6 +304,53 @@ VALUES (new.id, new.title, new.content); END; +-- Discourse forum topics +CREATE TABLE IF NOT EXISTS discourse_topics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + forum_url TEXT NOT NULL, + topic_id INTEGER NOT NULL, + title TEXT NOT NULL, + first_post TEXT, + accepted_answer TEXT, + category_name TEXT, + tags TEXT, + reply_count INTEGER DEFAULT 0, + like_count INTEGER DEFAULT 0, + views INTEGER DEFAULT 0, + url TEXT NOT NULL, + created_at TEXT NOT NULL, + last_posted_at TEXT, + synced_at TEXT NOT NULL, + UNIQUE(forum_url, topic_id) +); + +-- FTS5 for Discourse topic search +CREATE VIRTUAL TABLE IF NOT EXISTS discourse_topics_fts USING fts5( + title, + first_post, + accepted_answer, + content='discourse_topics', + content_rowid='id' +); + +-- Triggers to keep FTS in sync with discourse_topics +CREATE TRIGGER IF NOT EXISTS discourse_topics_ai AFTER INSERT ON discourse_topics BEGIN + INSERT INTO discourse_topics_fts(rowid, title, first_post, accepted_answer) + VALUES (new.id, new.title, new.first_post, new.accepted_answer); +END; + +CREATE TRIGGER IF NOT EXISTS discourse_topics_ad AFTER DELETE ON discourse_topics BEGIN + INSERT INTO discourse_topics_fts(discourse_topics_fts, rowid, title, first_post, accepted_answer) + VALUES('delete', old.id, old.title, old.first_post, old.accepted_answer); +END; + +CREATE TRIGGER IF NOT EXISTS discourse_topics_au AFTER UPDATE ON discourse_topics BEGIN + INSERT INTO discourse_topics_fts(discourse_topics_fts, rowid, title, first_post, accepted_answer) + VALUES('delete', old.id, old.title, old.first_post, old.accepted_answer); + INSERT INTO discourse_topics_fts(rowid, title, first_post, accepted_answer) + VALUES (new.id, new.title, new.first_post, new.accepted_answer); +END; + -- Indexes for efficient queries CREATE INDEX IF NOT EXISTS idx_github_items_repo ON github_items(repo); CREATE INDEX IF NOT EXISTS idx_github_items_status ON github_items(status); @@ -320,6 +367,9 @@ CREATE INDEX IF NOT EXISTS idx_faq_quality ON faq_entries(quality_score); CREATE INDEX IF NOT EXISTS idx_summarization_status ON summarization_status(list_name, status); CREATE INDEX IF NOT EXISTS idx_bep_status ON bep_items(status); +CREATE INDEX IF NOT EXISTS idx_discourse_forum ON discourse_topics(forum_url); +CREATE INDEX IF NOT EXISTS idx_discourse_category ON discourse_topics(category_name); +CREATE INDEX IF NOT EXISTS idx_discourse_created ON discourse_topics(created_at); """ @@ -565,8 +615,8 @@ def get_last_sync(source_type: str, source_name: str, project: str = "hed") -> s """Get last sync time for a source. Args: - source_type: 'github', 'papers', or 'beps' - source_name: Repository name, paper source name, or 'bids-website' + source_type: 'github', 'papers', 'beps', or 'discourse' + source_name: Repository name, paper source name, or base URL project: Assistant/project name. Defaults to 'hed'. Returns: @@ -586,8 +636,8 @@ def update_sync_metadata( """Update sync metadata for a source. Args: - source_type: 'github', 'papers', or 'beps' - source_name: Repository name, paper source name, or 'bids-website' + source_type: 'github', 'papers', 'beps', or 'discourse' + source_name: Repository name, paper source name, or base URL items_synced: Number of items synced in this run project: Assistant/project name. Defaults to 'hed'. """ @@ -728,6 +778,17 @@ def get_stats(project: str = "hed") -> dict[str, int]: else: raise + # Discourse stats (table may not exist in older databases) + try: + stats["discourse_total"] = conn.execute( + "SELECT COUNT(*) FROM discourse_topics" + ).fetchone()[0] + except sqlite3.OperationalError as e: + if "no such table" in str(e): + stats["discourse_total"] = 0 + else: + raise + return stats @@ -909,6 +970,85 @@ def update_summarization_status( ) +def upsert_discourse_topic( + conn: sqlite3.Connection, + *, + forum_url: str, + topic_id: int, + title: str, + first_post: str | None, + accepted_answer: str | None, + category_name: str | None, + tags: list[str] | None, + reply_count: int, + like_count: int, + views: int, + url: str, + created_at: str, + last_posted_at: str | None, +) -> None: + """Insert or update a Discourse forum topic. + + Args: + conn: Database connection + forum_url: Base URL of the Discourse instance + topic_id: Discourse topic ID + title: Topic title + first_post: Content of the first post (markdown) + accepted_answer: Content of the accepted answer (markdown), if any + category_name: Discourse category name + tags: List of topic tags + reply_count: Number of replies + like_count: Total likes on the topic + views: View count + url: Full URL to the topic + created_at: ISO 8601 creation timestamp + last_posted_at: ISO 8601 timestamp of last post + """ + # Limit post sizes to prevent bloat + if first_post and len(first_post) > 5000: + first_post = first_post[:5000] + if accepted_answer and len(accepted_answer) > 5000: + accepted_answer = accepted_answer[:5000] + + conn.execute( + """ + INSERT INTO discourse_topics (forum_url, topic_id, title, first_post, + accepted_answer, category_name, tags, + reply_count, like_count, views, url, + created_at, last_posted_at, synced_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(forum_url, topic_id) DO UPDATE SET + title=excluded.title, + first_post=excluded.first_post, + accepted_answer=excluded.accepted_answer, + category_name=excluded.category_name, + tags=excluded.tags, + reply_count=excluded.reply_count, + like_count=excluded.like_count, + views=excluded.views, + last_posted_at=excluded.last_posted_at, + synced_at=excluded.synced_at + """, + ( + forum_url, + topic_id, + title, + first_post, + accepted_answer, + category_name, + json.dumps(tags) if tags else None, + reply_count, + like_count, + views, + url, + created_at, + last_posted_at, + _now_iso(), + ), + ) + + def is_db_populated(project: str) -> dict[str, bool]: """Check which knowledge tables have data for a community. @@ -929,6 +1069,7 @@ def is_db_populated(project: str) -> dict[str, bool]: "mailman": "mailing_list_messages", "faq": "faq_entries", "beps": "bep_items", + "discourse": "discourse_topics", } db_path = get_db_path(project) diff --git a/src/knowledge/discourse_sync.py b/src/knowledge/discourse_sync.py new file mode 100644 index 0000000..89b81fb --- /dev/null +++ b/src/knowledge/discourse_sync.py @@ -0,0 +1,384 @@ +"""Discourse forum topic sync. + +Syncs topics from Discourse forums using the public JSON API. +Designed to be generic and work with any Discourse instance. + +Features: +- Public API (no auth needed for read access) +- Incremental sync (only new/updated topics since last sync) +- Category filtering +- Patient rate limiting (1 request per second by default) +- Stores topics in knowledge DB for FTS search +""" + +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING + +import httpx +import markdownify +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn + +if TYPE_CHECKING: + from src.core.config.community import DiscourseCategoryConfig + +from src.knowledge.db import ( + get_connection, + get_last_sync, + update_sync_metadata, + upsert_discourse_topic, +) + +logger = logging.getLogger(__name__) +console = Console() + +# Default delay between API requests (seconds). +# Discourse allows 200 req/min per IP, but we are generous and patient. +DEFAULT_REQUEST_DELAY = 1.0 + + +def _html_to_markdown(html: str) -> str: + """Convert Discourse post HTML to markdown.""" + if not html: + return "" + md = markdownify.markdownify(html, heading_style="ATX", strip=["script", "style"]) + # Collapse excessive whitespace + lines = [line.rstrip() for line in md.split("\n")] + cleaned = [] + blank_count = 0 + for line in lines: + if not line.strip(): + blank_count += 1 + if blank_count <= 2: + cleaned.append(line) + else: + blank_count = 0 + cleaned.append(line) + return "\n".join(cleaned).strip() + + +def _fetch_json( + url: str, + *, + timeout: float = 30.0, + delay: float = DEFAULT_REQUEST_DELAY, + max_retries: int = 3, +) -> dict | None: + """Fetch JSON from a URL with rate limiting and retry on 429. + + Args: + url: URL to fetch + timeout: HTTP timeout in seconds + delay: Delay after the request completes (rate limiting) + max_retries: Max retries on 429 Too Many Requests + + Returns: + Parsed JSON dict, or None on error + """ + for attempt in range(max_retries): + try: + response = httpx.get( + url, + timeout=timeout, + follow_redirects=True, + headers={"Accept": "application/json"}, + ) + if response.status_code == 429: + retry_after = int(response.headers.get("Retry-After", 10)) + logger.warning( + "Rate limited (429), waiting %ds (attempt %d)", retry_after, attempt + 1 + ) + time.sleep(retry_after) + continue + response.raise_for_status() + time.sleep(delay) + return response.json() + except httpx.HTTPStatusError as e: + logger.error("HTTP %d fetching %s: %s", e.response.status_code, url, e) + return None + except httpx.TimeoutException: + logger.error("Timeout fetching %s", url) + return None + except httpx.RequestError as e: + logger.error("Request error fetching %s: %s", url, e) + return None + + logger.error("Max retries exceeded for %s", url) + return None + + +def _get_accepted_answer(posts: list[dict]) -> str | None: + """Extract the accepted answer from a list of posts. + + Discourse marks accepted answers with 'accepted_answer' field. + Falls back to the most-liked reply if no accepted answer. + """ + # Look for the accepted answer + for post in posts: + if post.get("accepted_answer"): + return _html_to_markdown(post.get("cooked", "")) + + # Fall back to the reply with the most likes (skip OP which is post_number=1) + replies = [p for p in posts if p.get("post_number", 0) > 1] + if replies: + best = max(replies, key=lambda p: p.get("like_count", 0)) + if best.get("like_count", 0) > 0: + return _html_to_markdown(best.get("cooked", "")) + + return None + + +def sync_discourse_topics( + base_url: str, + project: str, + categories: list[DiscourseCategoryConfig] | None = None, + incremental: bool = True, + max_topics: int | None = None, + request_delay: float = DEFAULT_REQUEST_DELAY, +) -> int: + """Sync topics from a Discourse forum. + + Fetches topic listings and individual topic details from the Discourse + public JSON API. Stores topics with their first post and best answer + in the knowledge database. + + Args: + base_url: Base URL of the Discourse instance (e.g., 'https://mne.discourse.group') + project: Community ID for database isolation + categories: Optional list of category configs to limit sync to. + If None, syncs from /latest.json (all categories). + incremental: If True, only sync topics updated since last sync + max_topics: Maximum number of topics to sync (for testing). None for all. + request_delay: Seconds between API requests (default: 1.0s, patient) + + Returns: + Number of topics synced + """ + base_url = base_url.rstrip("/") + console.print(f"Syncing Discourse topics from {base_url}...") + + # Get last sync time for incremental sync + last_sync = None + if incremental: + last_sync = get_last_sync("discourse", base_url, project) + if last_sync: + console.print(f"Incremental sync since {last_sync}") + else: + console.print("No previous sync found, doing full sync") + + # Collect topic IDs to sync + topic_ids = _collect_topic_ids( + base_url, + categories=categories, + last_sync=last_sync, + max_topics=max_topics, + request_delay=request_delay, + ) + + if not topic_ids: + console.print("[yellow]No new topics to sync[/yellow]") + update_sync_metadata("discourse", base_url, 0, project) + return 0 + + console.print(f"Found {len(topic_ids)} topics to sync") + + # Fetch and store each topic + total_synced = 0 + failed = 0 + uncommitted = 0 + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Syncing topics...", total=len(topic_ids)) + + with get_connection(project) as conn: + for topic_id in topic_ids: + try: + topic_url = f"{base_url}/t/{topic_id}.json" + data = _fetch_json(topic_url, delay=request_delay) + + if data is None: + failed += 1 + progress.update(task, advance=1) + continue + + # Use .get() to avoid KeyError on malformed API responses + resolved_id = data.get("id", topic_id) + slug = data.get("slug", "") + + posts = data.get("post_stream", {}).get("posts", []) + first_post_html = posts[0].get("cooked", "") if posts else "" + first_post = _html_to_markdown(first_post_html) + accepted_answer = _get_accepted_answer(posts) if len(posts) > 1 else None + + upsert_discourse_topic( + conn, + forum_url=base_url, + topic_id=resolved_id, + title=data.get("title", ""), + first_post=first_post, + accepted_answer=accepted_answer, + category_name=data.get("category_name"), + tags=data.get("tags"), + reply_count=data.get("reply_count", 0), + like_count=data.get("like_count", 0), + views=data.get("views", 0), + url=f"{base_url}/t/{slug}/{resolved_id}", + created_at=data.get("created_at", ""), + last_posted_at=data.get("last_posted_at"), + ) + total_synced += 1 + uncommitted += 1 + + # Commit every 50 topics to avoid large transactions + if uncommitted >= 50: + conn.commit() + uncommitted = 0 + except Exception: + logger.exception("Failed to process topic %d from %s", topic_id, base_url) + failed += 1 + + progress.update(task, advance=1) + + # Final commit + conn.commit() + + # Update sync metadata + update_sync_metadata("discourse", base_url, total_synced, project) + + console.print(f"[green]Synced {total_synced} topics[/green]") + if failed: + console.print(f"[yellow]Failed to fetch {failed} topics[/yellow]") + + return total_synced + + +def _collect_topic_ids( + base_url: str, + *, + categories: list[DiscourseCategoryConfig] | None = None, + last_sync: str | None = None, + max_topics: int | None = None, + request_delay: float = DEFAULT_REQUEST_DELAY, +) -> list[int]: + """Collect topic IDs to sync from topic listings. + + Pages through /latest.json or category-specific listings to find + topics that need syncing. + + Args: + base_url: Discourse base URL + categories: Optional category filters + last_sync: ISO timestamp of last sync (for incremental) + max_topics: Maximum topics to collect + request_delay: Delay between requests + + Returns: + List of topic IDs to fetch + """ + topic_ids: list[int] = [] + + if categories: + # Sync specific categories + for cat in categories: + slug = cat.slug + cat_id = cat.id + ids = _collect_from_listing( + f"{base_url}/c/{slug}/{cat_id}.json", + last_sync=last_sync, + max_topics=max_topics - len(topic_ids) if max_topics else None, + request_delay=request_delay, + ) + topic_ids.extend(ids) + if max_topics and len(topic_ids) >= max_topics: + break + else: + # Sync all topics via latest + topic_ids = _collect_from_listing( + f"{base_url}/latest.json", + last_sync=last_sync, + max_topics=max_topics, + request_delay=request_delay, + ) + + return topic_ids[:max_topics] if max_topics else topic_ids + + +def _collect_from_listing( + url: str, + *, + last_sync: str | None = None, + max_topics: int | None = None, + request_delay: float = DEFAULT_REQUEST_DELAY, +) -> list[int]: + """Page through a Discourse topic listing and collect topic IDs. + + Args: + url: Listing URL (e.g., /latest.json or /c/slug/id.json) + last_sync: Stop collecting when we hit topics older than this + max_topics: Maximum topics to collect + request_delay: Delay between requests + + Returns: + List of topic IDs + """ + topic_ids: list[int] = [] + page = 0 + max_pages = 200 # Safety limit + + while page < max_pages: + page_url = f"{url}?page={page}" if page > 0 else url + data = _fetch_json(page_url, delay=request_delay) + + if data is None: + logger.warning( + "Listing fetch failed at page %d for %s; collected %d topics so far", + page, + url, + len(topic_ids), + ) + break + + topics = data.get("topic_list", {}).get("topics", []) + if not topics: + break + + hit_old_topics = False + for topic in topics: + # Skip pinned topics (they appear on every page) + if topic.get("pinned"): + continue + + topic_id = topic.get("id") + if topic_id is None: + continue + + # For incremental sync, stop at topics older than last_sync + if last_sync: + last_activity = topic.get("last_posted_at") or topic.get("created_at", "") + if last_activity and last_activity < last_sync: + hit_old_topics = True + break + + topic_ids.append(topic_id) + + if max_topics and len(topic_ids) >= max_topics: + return topic_ids + + if hit_old_topics: + break + + # Check if there are more pages + more_url = data.get("topic_list", {}).get("more_topics_url") + if not more_url: + break + + page += 1 + + return topic_ids diff --git a/src/knowledge/search.py b/src/knowledge/search.py index 30fc64e..4f6667c 100644 --- a/src/knowledge/search.py +++ b/src/knowledge/search.py @@ -770,3 +770,89 @@ def search_beps( raise return results + + +@dataclass +class DiscourseTopicResult: + """A Discourse forum topic search result.""" + + title: str + url: str + snippet: str + category_name: str + reply_count: int + like_count: int + views: int + accepted_answer_snippet: str | None + created_at: str + + +def search_discourse_topics( + query: str, + project: str = "mne", + limit: int = 5, + category_name: str | None = None, +) -> list[DiscourseTopicResult]: + """Search Discourse forum topics using full-text search. + + Args: + query: Search phrase + project: Community ID for database isolation. Defaults to 'mne'. + limit: Maximum number of results + category_name: Filter by Discourse category name + + Returns: + List of matching topics, ordered by relevance + """ + sql = """ + SELECT d.title, d.url, d.first_post, d.accepted_answer, + d.category_name, d.reply_count, d.like_count, d.views, + d.created_at + FROM discourse_topics_fts fts + JOIN discourse_topics d ON fts.rowid = d.id + WHERE discourse_topics_fts MATCH ? + """ + params: list[str | int] = [query] + + if category_name: + sql += " AND d.category_name = ?" + params.append(category_name) + + sql += " ORDER BY rank LIMIT ?" + params.append(limit) + + results = [] + try: + with get_connection(project) as conn: + safe_query = _sanitize_fts5_query(query) + params[0] = safe_query + + for row in conn.execute(sql, params): + results.append( + DiscourseTopicResult( + title=row["title"], + url=row["url"], + snippet=_make_snippet(row["first_post"], max_length=300), + category_name=row["category_name"] or "", + reply_count=row["reply_count"], + like_count=row["like_count"], + views=row["views"], + accepted_answer_snippet=( + _make_snippet(row["accepted_answer"], max_length=200) or None + ), + created_at=row["created_at"] or "", + ) + ) + except sqlite3.OperationalError as e: + logger.error( + "Database operational error during Discourse search: %s", + e, + exc_info=True, + extra={"query": query, "project": project}, + ) + raise + except sqlite3.Error as e: + logger.warning("Database error during Discourse search '%s': %s", query, e) + raise + + return results diff --git a/src/tools/fetcher.py b/src/tools/fetcher.py index 3fd6f86..e45dea5 100644 --- a/src/tools/fetcher.py +++ b/src/tools/fetcher.py @@ -1,6 +1,8 @@ """Document fetching utility with caching for OSA tools.""" import hashlib +import logging +import re import time from dataclasses import dataclass, field from pathlib import Path @@ -11,6 +13,67 @@ from src.tools.base import DocPage, RetrievedDoc from src.tools.markdown_cleaner import clean_markdown +logger = logging.getLogger(__name__) + +# Selectors for extracting main content from HTML pages, in priority order. +# Covers Sphinx (PyData theme, RTD theme, Furo), MkDocs, and generic HTML. +_CONTENT_SELECTORS = [ + "article.bd-article", # PyData Sphinx theme + "div[role=main]", # Read the Docs / classic Sphinx + "article[role=main]", # Furo Sphinx theme + "main", # Generic HTML5 + "div.document", # Older Sphinx + "div.md-content", # MkDocs Material +] + + +def _is_html(content: str) -> bool: + """Check if content appears to be HTML.""" + stripped = content.lstrip() + return stripped.startswith((" str: + """Convert HTML to markdown, extracting main content if possible. + + Uses BeautifulSoup to find the main content area (skipping nav, sidebar, + footer), then markdownify for HTML-to-markdown conversion. + + Requires beautifulsoup4 and markdownify (server optional dependencies). + """ + import markdownify + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html, "html.parser") + + # Try to find the main content area + content_element = None + for selector in _CONTENT_SELECTORS: + content_element = soup.select_one(selector) + if content_element: + break + + # Fall back to full body if no content area found + if content_element is None: + content_element = soup.body or soup + + # Remove nav, sidebar, footer elements within content + for tag in content_element.find_all(["nav", "footer", "aside"]): + tag.decompose() + + md = markdownify.markdownify( + str(content_element), + heading_style="ATX", + strip=["script", "style"], + ) + + # Clean up Sphinx anchor links like [#](#heading "Link to this heading") + md = re.sub(r'\[#\]\([^)]*"Link to this [^"]*"\)', "", md) + # Clean up bare anchor links [#](#id) + md = re.sub(r"\[#\]\(#[^)]*\)", "", md) + + return md + @dataclass class CacheEntry: @@ -179,7 +242,14 @@ def fetch(self, doc: DocPage) -> RetrievedDoc: response.raise_for_status() content = response.text - # Cache the raw content (before cleaning) + # Convert HTML to markdown before caching + if _is_html(content): + logger.debug( + "Detected HTML content, converting to markdown: %s", doc.source_url + ) + content = _html_to_markdown(content) + + # Cache the content (after HTML conversion, before markdown cleaning) self._save_to_cache(doc.source_url, content) # Clean markdown if enabled diff --git a/src/tools/knowledge.py b/src/tools/knowledge.py index cb213f3..6729e71 100644 --- a/src/tools/knowledge.py +++ b/src/tools/knowledge.py @@ -24,6 +24,7 @@ from src.knowledge.db import get_db_path from src.knowledge.search import ( list_recent_github_items, + search_discourse_topics, search_docstrings, search_github_items, search_papers, @@ -417,6 +418,84 @@ def search_faq_impl( ) +def create_search_discourse_tool( + community_id: str, + community_name: str, +) -> BaseTool: + """Create a tool for searching Discourse forum topics. + + Args: + community_id: The community identifier (e.g., 'mne') + community_name: Display name (e.g., 'MNE-Python') + + Returns: + A LangChain tool for searching Discourse forum topics + """ + + def search_discourse_impl( + query: str, + category: str | None = None, + limit: int = 5, + ) -> str: + """Search Discourse forum topics implementation.""" + if not _check_db_exists(community_id): + return ( + f"Knowledge database for {community_name} not initialized. " + "Run 'osa sync discourse' to populate it." + ) + + try: + results = search_discourse_topics( + query=query, + project=community_id, + limit=limit, + category_name=category, + ) + except sqlite3.OperationalError as e: + if "no such table" in str(e): + logger.warning( + "Discourse table not initialized for %s", + community_id, + extra={"query": query, "community": community_id}, + ) + return ( + f"Discourse database for {community_name} not initialized. " + f"Run 'osa sync discourse --community {community_id}' to populate it." + ) + raise + + if not results: + cat_str = f" (category: {category})" if category else "" + return f"No forum topics found for '{query}'{cat_str}." + + lines = [f"Found {len(results)} forum topics:\n"] + for i, r in enumerate(results, 1): + cat_label = f" [{r.category_name}]" if r.category_name else "" + lines.append(f"**{i}. {r.title}**{cat_label}") + lines.append(f" Replies: {r.reply_count} | Likes: {r.like_count} | Views: {r.views}") + if r.snippet: + lines.append(f" {r.snippet}") + if r.accepted_answer_snippet: + lines.append(f" Accepted answer: {r.accepted_answer_snippet}") + lines.append(f" [View topic]({r.url})\n") + + return "\n".join(lines) + + description = ( + f"Search {community_name} Discourse forum topics for community discussions and Q&A. " + "**IMPORTANT: This is for DISCOVERY, not answering.** " + "Use this to find forum discussions where users have asked similar questions. " + 'Present results as: "There\'s a related discussion on the forum, see: [link]" ' + "Do NOT use forum content to formulate authoritative answers." + ) + + return StructuredTool.from_function( + func=search_discourse_impl, + name=f"search_{community_id}_forum", + description=description, + ) + + def create_knowledge_tools( community_id: str, community_name: str, @@ -428,6 +507,7 @@ def create_knowledge_tools( docstrings_language: str | None = None, include_faq: bool = False, faq_list_names: list[str] | None = None, + include_discourse: bool = False, ) -> list[BaseTool]: """Create all knowledge discovery tools for a community. @@ -445,6 +525,7 @@ def create_knowledge_tools( docstrings_language: Filter docstrings by language ('matlab' or 'python') include_faq: Include mailing list FAQ search tool (default: False) faq_list_names: List of mailing list names for FAQ help text + include_discourse: Include Discourse forum search tool (default: False) Returns: List of LangChain tools for the community @@ -468,4 +549,7 @@ def create_knowledge_tools( if include_faq: tools.append(create_search_faq_tool(community_id, community_name, faq_list_names)) + if include_discourse: + tools.append(create_search_discourse_tool(community_id, community_name)) + return tools diff --git a/tests/test_knowledge/test_db.py b/tests/test_knowledge/test_db.py index 561e047..12b2e00 100644 --- a/tests/test_knowledge/test_db.py +++ b/tests/test_knowledge/test_db.py @@ -367,7 +367,7 @@ def test_nonexistent_db(self, tmp_path: Path): with patch("src.knowledge.db.get_db_path", return_value=db_path): result = is_db_populated("nonexistent") assert all(v is False for v in result.values()) - expected_keys = {"github", "papers", "docstrings", "mailman", "faq", "beps"} + expected_keys = {"github", "papers", "docstrings", "mailman", "faq", "beps", "discourse"} assert set(result.keys()) == expected_keys def test_empty_db(self, temp_db: Path): diff --git a/tests/test_knowledge/test_discourse_sync.py b/tests/test_knowledge/test_discourse_sync.py new file mode 100644 index 0000000..e7df605 --- /dev/null +++ b/tests/test_knowledge/test_discourse_sync.py @@ -0,0 +1,371 @@ +"""Tests for the Discourse forum sync and search. + +Tests cover: +- DB schema creation and topic upsert +- FTS5 search on discourse topics +- Config validation (MNE community) +- Live Discourse API fetch (against mne.discourse.group) +""" + +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest + +from src.knowledge.db import ( + get_connection, + init_db, + upsert_discourse_topic, +) +from src.knowledge.search import DiscourseTopicResult, search_discourse_topics + + +@pytest.fixture +def temp_db(tmp_path: Path): + """Create a temporary database for testing.""" + db_path = tmp_path / "knowledge" / "test_discourse.db" + + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + yield db_path + + +class TestDiscourseDbSchema: + """Tests for Discourse database schema and upsert.""" + + def test_discourse_table_exists(self, temp_db: Path): + """Test that discourse_topics table is created.""" + conn = sqlite3.connect(temp_db) + tables = [ + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + ] + conn.close() + assert "discourse_topics" in tables + assert "discourse_topics_fts" in tables + + def test_upsert_discourse_topic(self, temp_db: Path): + """Test inserting and updating a discourse topic.""" + with patch("src.knowledge.db.get_db_path", return_value=temp_db), get_connection() as conn: + upsert_discourse_topic( + conn, + forum_url="https://mne.discourse.group", + topic_id=123, + title="How to read EDF files", + first_post="I want to read EDF files using MNE-Python.", + accepted_answer="Use mne.io.read_raw_edf().", + category_name="Support", + tags=["edf", "io"], + reply_count=5, + like_count=3, + views=100, + url="https://mne.discourse.group/t/how-to-read-edf-files/123", + created_at="2024-01-15T10:00:00Z", + last_posted_at="2024-01-16T14:00:00Z", + ) + conn.commit() + + # Verify the topic was inserted + row = conn.execute( + "SELECT title, first_post, accepted_answer, category_name " + "FROM discourse_topics WHERE topic_id = 123" + ).fetchone() + assert row is not None + assert row[0] == "How to read EDF files" + assert row[1] == "I want to read EDF files using MNE-Python." + assert row[2] == "Use mne.io.read_raw_edf()." + assert row[3] == "Support" + + def test_upsert_updates_existing(self, temp_db: Path): + """Test that upsert updates an existing topic.""" + with patch("src.knowledge.db.get_db_path", return_value=temp_db), get_connection() as conn: + # Insert first + upsert_discourse_topic( + conn, + forum_url="https://mne.discourse.group", + topic_id=456, + title="Original title", + first_post="Original post", + accepted_answer=None, + category_name=None, + tags=None, + reply_count=0, + like_count=0, + views=10, + url="https://mne.discourse.group/t/original/456", + created_at="2024-01-01T00:00:00Z", + last_posted_at=None, + ) + conn.commit() + + # Update + upsert_discourse_topic( + conn, + forum_url="https://mne.discourse.group", + topic_id=456, + title="Updated title", + first_post="Updated post", + accepted_answer="New answer", + category_name="General", + tags=["test"], + reply_count=10, + like_count=5, + views=200, + url="https://mne.discourse.group/t/updated/456", + created_at="2024-01-01T00:00:00Z", + last_posted_at="2024-02-01T00:00:00Z", + ) + conn.commit() + + row = conn.execute( + "SELECT title, reply_count, accepted_answer " + "FROM discourse_topics WHERE topic_id = 456" + ).fetchone() + assert row[0] == "Updated title" + assert row[1] == 10 + assert row[2] == "New answer" + + # Verify only one row exists + count = conn.execute( + "SELECT COUNT(*) FROM discourse_topics WHERE topic_id = 456" + ).fetchone()[0] + assert count == 1 + + +class TestDiscourseSearch: + """Tests for FTS5 search on discourse topics.""" + + def test_search_finds_topic(self, temp_db: Path): + """Test that search finds indexed topics.""" + with patch("src.knowledge.db.get_db_path", return_value=temp_db): + with get_connection() as conn: + upsert_discourse_topic( + conn, + forum_url="https://mne.discourse.group", + topic_id=789, + title="Epoch rejection threshold", + first_post="What is the best threshold for epoch rejection in MNE?", + accepted_answer="Use autoreject or set reject dict manually.", + category_name="Support", + tags=["epochs", "rejection"], + reply_count=8, + like_count=4, + views=250, + url="https://mne.discourse.group/t/epoch-rejection/789", + created_at="2024-03-01T00:00:00Z", + last_posted_at="2024-03-02T00:00:00Z", + ) + conn.commit() + + results = search_discourse_topics("epoch rejection", project="test_discourse", limit=5) + assert len(results) >= 1 + assert isinstance(results[0], DiscourseTopicResult) + assert "Epoch rejection" in results[0].title + assert results[0].reply_count == 8 + + def test_search_empty_query_returns_empty(self, temp_db: Path): + """Test that an empty or nonsensical query returns no results.""" + with patch("src.knowledge.db.get_db_path", return_value=temp_db): + results = search_discourse_topics( + "xyznonexistent12345", project="test_discourse", limit=5 + ) + assert results == [] + + +class TestHtmlToMarkdown: + """Tests for _html_to_markdown helper.""" + + def test_simple_html(self): + """Should convert simple HTML to plain text.""" + from src.knowledge.discourse_sync import _html_to_markdown + + result = _html_to_markdown("

Hello world

") + assert "Hello" in result + assert "world" in result + + def test_empty_input(self): + """Should return empty string for empty input.""" + from src.knowledge.discourse_sync import _html_to_markdown + + assert _html_to_markdown("") == "" + assert _html_to_markdown(None) == "" + + def test_code_blocks_preserved(self): + """Should preserve code block content.""" + from src.knowledge.discourse_sync import _html_to_markdown + + html = "

Use this:

mne.io.read_raw_edf('file.edf')
" + result = _html_to_markdown(html) + assert "read_raw_edf" in result + + def test_collapses_excessive_whitespace(self): + """Should collapse more than 2 consecutive blank lines.""" + from src.knowledge.discourse_sync import _html_to_markdown + + html = "

Line 1

\n\n\n\n\n\n

Line 2

" + result = _html_to_markdown(html) + # Should not have more than 2 consecutive blank lines + assert "\n\n\n\n" not in result + + +class TestGetAcceptedAnswer: + """Tests for _get_accepted_answer helper.""" + + def test_finds_accepted_answer(self): + """Should return the accepted answer post.""" + from src.knowledge.discourse_sync import _get_accepted_answer + + posts = [ + {"post_number": 1, "cooked": "

Question

", "accepted_answer": False}, + {"post_number": 2, "cooked": "

Wrong answer

", "like_count": 1}, + {"post_number": 3, "cooked": "

Correct answer

", "accepted_answer": True}, + ] + result = _get_accepted_answer(posts) + assert result is not None + assert "Correct answer" in result + + def test_falls_back_to_most_liked(self): + """Should fall back to highest-liked reply when no accepted answer.""" + from src.knowledge.discourse_sync import _get_accepted_answer + + posts = [ + {"post_number": 1, "cooked": "

Question

"}, + {"post_number": 2, "cooked": "

Good answer

", "like_count": 5}, + {"post_number": 3, "cooked": "

OK answer

", "like_count": 2}, + ] + result = _get_accepted_answer(posts) + assert result is not None + assert "Good answer" in result + + def test_returns_none_when_no_replies(self): + """Should return None when only OP exists.""" + from src.knowledge.discourse_sync import _get_accepted_answer + + posts = [{"post_number": 1, "cooked": "

Question

"}] + result = _get_accepted_answer(posts) + assert result is None + + def test_returns_none_when_no_liked_replies(self): + """Should return None when replies have zero likes.""" + from src.knowledge.discourse_sync import _get_accepted_answer + + posts = [ + {"post_number": 1, "cooked": "

Question

"}, + {"post_number": 2, "cooked": "

Reply

", "like_count": 0}, + ] + result = _get_accepted_answer(posts) + assert result is None + + +class TestMNEConfig: + """Tests for MNE community configuration.""" + + def test_mne_config_loads(self): + """Test that MNE config.yaml loads and validates correctly.""" + from src.core.config.community import CommunityConfig + + config = CommunityConfig.from_yaml("src/assistants/mne/config.yaml") + assert config.id == "mne" + assert config.name == "MNE-Python" + assert len(config.documentation) > 0 + assert len(config.discourse) == 1 + assert "mne.discourse.group" in str(config.discourse[0].url) + + def test_mne_has_github_repos(self): + """Test that MNE config has GitHub repos configured.""" + from src.core.config.community import CommunityConfig + + config = CommunityConfig.from_yaml("src/assistants/mne/config.yaml") + assert config.github is not None + assert len(config.github.repos) >= 5 + assert "mne-tools/mne-python" in config.github.repos + + def test_mne_has_docstrings(self): + """Test that MNE config has docstring repos configured.""" + from src.core.config.community import CommunityConfig + + config = CommunityConfig.from_yaml("src/assistants/mne/config.yaml") + assert config.docstrings is not None + assert len(config.docstrings.repos) >= 5 + repo_names = [r.repo for r in config.docstrings.repos] + assert "mne-tools/mne-python" in repo_names + + def test_mne_has_citations(self): + """Test that MNE config has citation DOIs.""" + from src.core.config.community import CommunityConfig + + config = CommunityConfig.from_yaml("src/assistants/mne/config.yaml") + assert config.citations is not None + assert len(config.citations.dois) >= 5 + + def test_mne_has_sync_schedule(self): + """Test that MNE config has sync schedules configured.""" + from src.core.config.community import CommunityConfig + + config = CommunityConfig.from_yaml("src/assistants/mne/config.yaml") + assert config.sync is not None + assert config.sync.discourse is not None + assert config.sync.github is not None + + +class TestDiscourseApiLive: + """Live tests against mne.discourse.group public API. + + These tests make real HTTP requests. They verify the Discourse + API integration works end-to-end. + """ + + @pytest.mark.network + def test_fetch_latest_topics(self): + """Test fetching latest topics from MNE Discourse.""" + from src.knowledge.discourse_sync import _fetch_json + + data = _fetch_json("https://mne.discourse.group/latest.json", delay=0.5) + assert data is not None + topics = data.get("topic_list", {}).get("topics", []) + assert len(topics) > 0 + # Each topic should have an id and title + first = topics[0] + assert "id" in first + assert "title" in first + + @pytest.mark.network + def test_fetch_single_topic(self): + """Test fetching a single topic with posts.""" + from src.knowledge.discourse_sync import _fetch_json + + # First get a valid topic ID from latest + latest = _fetch_json("https://mne.discourse.group/latest.json", delay=0.5) + assert latest is not None + topics = latest["topic_list"]["topics"] + assert len(topics) > 0 + topic_id = topics[0]["id"] + + # Now fetch that specific topic + data = _fetch_json(f"https://mne.discourse.group/t/{topic_id}.json", delay=0.5) + assert data is not None + assert "title" in data + posts = data.get("post_stream", {}).get("posts", []) + assert len(posts) >= 1 + + @pytest.mark.network + def test_sync_small_batch(self, temp_db: Path): + """Test syncing a small batch of topics end-to-end.""" + from src.knowledge.discourse_sync import sync_discourse_topics + + with patch("src.knowledge.db.get_db_path", return_value=temp_db): + count = sync_discourse_topics( + base_url="https://mne.discourse.group", + project="test_discourse", + incremental=False, + max_topics=3, + request_delay=0.5, + ) + assert count >= 1 + assert count <= 3 + + # Verify topics are searchable + with get_connection("test_discourse") as conn: + rows = conn.execute("SELECT COUNT(*) FROM discourse_topics").fetchone() + assert rows[0] >= 1 diff --git a/tests/test_tools/test_knowledge_tools.py b/tests/test_tools/test_knowledge_tools.py index 46d90df..e509abb 100644 --- a/tests/test_tools/test_knowledge_tools.py +++ b/tests/test_tools/test_knowledge_tools.py @@ -14,6 +14,7 @@ from src.tools.knowledge import ( create_knowledge_tools, create_list_recent_tool, + create_search_discourse_tool, create_search_discussions_tool, create_search_docstrings_tool, create_search_faq_tool, @@ -281,13 +282,23 @@ def test_includes_faq_tool_when_enabled(self) -> None: assert "search_test_faq" in tool_names assert len(tools) == 4 + def test_includes_discourse_tool_when_enabled(self) -> None: + """Should include Discourse forum search tool when include_discourse=True.""" + tools = create_knowledge_tools("test", "Test", include_discourse=True) + tool_names = [t.name for t in tools] + assert "search_test_forum" in tool_names + assert len(tools) == 4 + def test_includes_all_optional_tools(self) -> None: """Should include all tools when all flags enabled.""" - tools = create_knowledge_tools("test", "Test", include_docstrings=True, include_faq=True) + tools = create_knowledge_tools( + "test", "Test", include_docstrings=True, include_faq=True, include_discourse=True + ) tool_names = [t.name for t in tools] assert "search_test_code_docs" in tool_names assert "search_test_faq" in tool_names - assert len(tools) == 5 + assert "search_test_forum" in tool_names + assert len(tools) == 6 class TestSearchDocstringsTool: @@ -332,6 +343,46 @@ def test_handles_missing_table(self, tmp_path: Path) -> None: assert "osa sync mailman" in result +class TestSearchDiscourseTool: + """Tests for Discourse forum search tool.""" + + def test_handles_missing_table(self, tmp_path: Path) -> None: + """Should return friendly message when discourse_topics table doesn't exist.""" + import sqlite3 + + tool = create_search_discourse_tool("test", "Test Community") + + db_path = tmp_path / "knowledge" / "test.db" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.execute("CREATE TABLE dummy (id INTEGER)") + conn.close() + + with patch("src.tools.knowledge.get_db_path", return_value=db_path): + result = tool.invoke({"query": "epoch rejection"}) + assert "not initialized" in result.lower() + assert "osa sync discourse" in result + + def test_returns_no_results_message(self, tmp_path: Path) -> None: + """Should return 'no results' message for non-matching query.""" + tool = create_search_discourse_tool("test", "Test Community") + + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db("test") + with patch("src.tools.knowledge.get_db_path", return_value=db_path): + result = tool.invoke({"query": "xyznonexistent123"}) + assert "No forum topics found" in result + + def test_tool_has_correct_name(self) -> None: + """Tool should have community-specific name.""" + tool = create_search_discourse_tool("hed", "HED") + assert tool.name == "search_hed_forum" + + tool = create_search_discourse_tool("mne", "MNE-Python") + assert tool.name == "search_mne_forum" + + class TestHEDKnowledgeToolsIntegration: """Integration tests for HED knowledge tools via registry.""" From ebef53ea722a302eca8b777eb23c289a32aa5d7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 25 Feb 2026 12:12:31 +0000 Subject: [PATCH 24/26] chore: sync worker CORS from community configs [skip ci] --- workers/osa-worker/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/workers/osa-worker/index.js b/workers/osa-worker/index.js index 1e5ef0a..9fb5ea3 100644 --- a/workers/osa-worker/index.js +++ b/workers/osa-worker/index.js @@ -157,12 +157,14 @@ function isAllowedOrigin(origin) { // Allowed origins for OSA const allowedPatterns = [ 'https://osc.earth', + 'https://*.mne.tools', 'https://bids-specification.readthedocs.io', 'https://bids.neuroimaging.io', 'https://eeglab.org', 'https://fieldtriptoolbox.org', 'https://hedtags.org', 'https://hedtools.org', + 'https://mne.tools', 'https://nemar.org', 'https://sccn.github.io', 'https://www.eeglab.org', @@ -181,6 +183,7 @@ function isAllowedOrigin(origin) { if (origin.startsWith('https://') && origin.endsWith('.github.io')) return true; if (origin.startsWith('https://') && origin.endsWith('.hedtags.org')) return true; if (origin.startsWith('https://') && origin.endsWith('.hedtools.org')) return true; + if (origin.startsWith('https://') && origin.endsWith('.mne.tools')) return true; if (origin.startsWith('https://') && origin.endsWith('.nemar.org')) return true; if (origin.startsWith('https://') && origin.endsWith('.neuroimaging.io')) return true; if (origin.startsWith('https://') && origin.endsWith('.readthedocs.io')) return true; From 5d68353b963739713fbcb5ee1ec6635dcca45feb Mon Sep 17 00:00:00 2001 From: "Seyed (Yahya) Shirazi" Date: Wed, 25 Feb 2026 04:43:00 -0800 Subject: [PATCH 25/26] Remove CORS origins for MNE and FieldTrip (#231) Not yet onboarded by their maintainers; only hosted on demo.osc.earth for now. Left TODO comments for maintainers to submit PRs when ready. --- src/assistants/fieldtrip/config.yaml | 7 ++++--- src/assistants/mne/config.yaml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/assistants/fieldtrip/config.yaml b/src/assistants/fieldtrip/config.yaml index 898e3cb..276d6ad 100644 --- a/src/assistants/fieldtrip/config.yaml +++ b/src/assistants/fieldtrip/config.yaml @@ -28,9 +28,10 @@ widget: - How do I do beamforming source analysis? - What data formats does FieldTrip support? -cors_origins: - - https://www.fieldtriptoolbox.org - - https://fieldtriptoolbox.org +# TODO: FieldTrip maintainers to submit PR adding CORS origins for fieldtriptoolbox.org +# cors_origins: +# - https://www.fieldtriptoolbox.org +# - https://fieldtriptoolbox.org # Budget limits for cost management budget: diff --git a/src/assistants/mne/config.yaml b/src/assistants/mne/config.yaml index b2aa590..2000c36 100644 --- a/src/assistants/mne/config.yaml +++ b/src/assistants/mne/config.yaml @@ -28,9 +28,10 @@ widget: - How do I perform source localization? - How do I do time-frequency analysis? -cors_origins: - - https://mne.tools - - https://*.mne.tools +# TODO: MNE maintainers to submit PR adding CORS origins for mne.tools +# cors_origins: +# - https://mne.tools +# - https://*.mne.tools # Budget limits for cost management budget: From 20064e9402135ee696093dd028202e0034d9d317 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 25 Feb 2026 12:43:11 +0000 Subject: [PATCH 26/26] chore: sync worker CORS from community configs [skip ci] --- workers/osa-worker/index.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/workers/osa-worker/index.js b/workers/osa-worker/index.js index 9fb5ea3..cdf066c 100644 --- a/workers/osa-worker/index.js +++ b/workers/osa-worker/index.js @@ -157,18 +157,14 @@ function isAllowedOrigin(origin) { // Allowed origins for OSA const allowedPatterns = [ 'https://osc.earth', - 'https://*.mne.tools', 'https://bids-specification.readthedocs.io', 'https://bids.neuroimaging.io', 'https://eeglab.org', - 'https://fieldtriptoolbox.org', 'https://hedtags.org', 'https://hedtools.org', - 'https://mne.tools', 'https://nemar.org', 'https://sccn.github.io', 'https://www.eeglab.org', - 'https://www.fieldtriptoolbox.org', 'https://www.hedtags.org', 'https://www.hedtools.org', 'https://www.nemar.org' @@ -179,11 +175,9 @@ function isAllowedOrigin(origin) { // Check subdomains if (origin.startsWith('https://') && origin.endsWith('.eeglab.org')) return true; - if (origin.startsWith('https://') && origin.endsWith('.fieldtriptoolbox.org')) return true; if (origin.startsWith('https://') && origin.endsWith('.github.io')) return true; if (origin.startsWith('https://') && origin.endsWith('.hedtags.org')) return true; if (origin.startsWith('https://') && origin.endsWith('.hedtools.org')) return true; - if (origin.startsWith('https://') && origin.endsWith('.mne.tools')) return true; if (origin.startsWith('https://') && origin.endsWith('.nemar.org')) return true; if (origin.startsWith('https://') && origin.endsWith('.neuroimaging.io')) return true; if (origin.startsWith('https://') && origin.endsWith('.readthedocs.io')) return true;