From 4c9b180431f81e42c1c3c5c98a4263edaa6025a9 Mon Sep 17 00:00:00 2001 From: Andrew Sliva Date: Mon, 2 Mar 2026 12:08:47 -0700 Subject: [PATCH 1/3] fix: improve reaction handling logic in Discord bot --- systems/docgpt/src/app/discord.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/systems/docgpt/src/app/discord.py b/systems/docgpt/src/app/discord.py index 21d21db..202a687 100644 --- a/systems/docgpt/src/app/discord.py +++ b/systems/docgpt/src/app/discord.py @@ -147,8 +147,20 @@ async def on_message( add_start_index=True, ).split_text(result.answer) + first_reply_message: discord.Message | None = None for reply in response_chunks: - await user_message.reply(reply) + sent = await user_message.reply(reply) + if first_reply_message is None: + first_reply_message = sent + + # Add feedback reactions to the assistant's first reply (the answer), + # not to the original user question message. + if first_reply_message is not None: + try: + await first_reply_message.add_reaction("👍") + await first_reply_message.add_reaction("👎") + except Exception: + log.exception("Failed to add feedback reactions to assistant reply") if channel.name.lower() == NEW_THREAD_NAME.lower(): title_result = assistant.prompt( From 2cdc4f2328ef0364a3e4801c39c1b0ff54db6f36 Mon Sep 17 00:00:00 2001 From: Andrew Sliva Date: Mon, 23 Mar 2026 11:37:21 -0700 Subject: [PATCH 2/3] add test environment config and related scripts for test infrastructure --- systems/docgpt/.env.test.example | 18 +++++++ systems/docgpt/README.md | 67 ++++++++++++++++++++++++++ systems/docgpt/docker-compose.test.yml | 31 ++++++++++++ systems/docgpt/run-test-bot.sh | 18 +++++++ 4 files changed, 134 insertions(+) create mode 100644 systems/docgpt/.env.test.example create mode 100644 systems/docgpt/docker-compose.test.yml create mode 100644 systems/docgpt/run-test-bot.sh diff --git a/systems/docgpt/.env.test.example b/systems/docgpt/.env.test.example new file mode 100644 index 0000000..360e6b0 --- /dev/null +++ b/systems/docgpt/.env.test.example @@ -0,0 +1,18 @@ +# Gemini API key (or set GOOGLE_API_KEY) +AI_GEMINI_APIKEY= + +# Discord bot token (use a dedicated test bot token) +APP_DISCORD_TOKEN= + +# Force pgvector backend for isolated test database runs +STORAGE_VECTOR_BACKEND=pgvector + +# PostgreSQL test database (docker-compose.test.yml) +# Format: postgresql+psycopg://USER:PASSWORD@HOST:PORT/DATABASE +STORAGE_VECTOR_URL=postgresql+psycopg://root:example@localhost:55432/docgpt_test + +# MongoDB test database (docker-compose.test.yml) +STORAGE_MEMORY_URL=mongodb://root:example@localhost:27018/docgpt_test + +# Optional +LOG_LEVEL=INFO diff --git a/systems/docgpt/README.md b/systems/docgpt/README.md index 2bbd56f..7e22a9d 100644 --- a/systems/docgpt/README.md +++ b/systems/docgpt/README.md @@ -25,3 +25,70 @@ Ask about the R data.table package documentation and contribution guide. 2. Run ```docker compose up``` 3. Ingest data (once): ```uv run python main.py --ingest``` 4. Start the Discord bot: ```uv run python main.py``` + +## Manual bot testing with separate test DB + +Use this flow when you want to interact with the bot in Discord without touching normal dev data. + +1. Create a test env file from the template: + ```powershell + Copy-Item .env.test.example .env.test + ``` +2. Fill in `AI_GEMINI_APIKEY` and `APP_DISCORD_TOKEN` in `.env.test`. +3. Start isolated test databases: + ```powershell + docker compose -f docker-compose.test.yml up -d + ``` +4. Load `.env.test` into the current PowerShell session: + ```powershell + Get-Content .env.test | ForEach-Object { + if ($_ -match '^\s*#' -or $_ -match '^\s*$') { return } + $name, $value = $_ -split '=', 2 + Set-Item -Path "Env:$name" -Value $value + } + ``` +5. Ingest documents into the test vector database: + ```powershell + uv run python main.py --ingest + ``` +6. Run the Discord bot using the test DB settings: + ```powershell + uv run python main.py + ``` +7. Tear down and wipe test data when done: + ```powershell + docker compose -f docker-compose.test.yml down -v + ``` + +This keeps vector data and chat memory isolated to test services (`localhost:55432`, `localhost:27018`) and removes persisted test data on teardown. + +## Run a second bot on EC2 (side-by-side with prod) + +Use this when your production bot is already running and you want a separate test bot process. + +1. Create a second Discord bot application/token (test-only) and add it to a test server. +2. Prepare test environment values: + ```bash + cp .env.test.example .env.test + ``` +3. Edit `.env.test` and set: + - `APP_DISCORD_TOKEN` to the test bot token + - `AI_GEMINI_APIKEY` +4. Ingest test data once: + ```bash + set -a && source .env.test && set +a + docker compose -f docker-compose.test.yml up -d + uv run python main.py --ingest + ``` +5. Run the second bot: + ```bash + ./run-test-bot.sh + ``` + +Notes: +- Do not reuse the production bot token for the test bot. +- Production bot keeps using `.env`; test bot uses `.env.test`. +- Stop and wipe test data when finished: + ```bash + docker compose -f docker-compose.test.yml down -v + ``` diff --git a/systems/docgpt/docker-compose.test.yml b/systems/docgpt/docker-compose.test.yml new file mode 100644 index 0000000..fe3794f --- /dev/null +++ b/systems/docgpt/docker-compose.test.yml @@ -0,0 +1,31 @@ +version: "3" + +services: + vector_storage_test: + image: ankane/pgvector + container_name: vector_storage_test + restart: always + ports: + - "55432:5432" + environment: + - POSTGRES_USER=root + - POSTGRES_PASSWORD=example + - POSTGRES_DB=docgpt_test + volumes: + - docgpt_test_postgres_data:/var/lib/postgresql/data + + memory_storage_test: + image: mongo + container_name: memory_storage_test + restart: always + ports: + - "27018:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + volumes: + - docgpt_test_mongo_data:/data/db + +volumes: + docgpt_test_postgres_data: + docgpt_test_mongo_data: diff --git a/systems/docgpt/run-test-bot.sh b/systems/docgpt/run-test-bot.sh new file mode 100644 index 0000000..68af4a9 --- /dev/null +++ b/systems/docgpt/run-test-bot.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ! -f ".env.test" ]]; then + echo ".env.test not found. Create it from .env.test.example first." + exit 1 +fi + +# Export all variables from .env.test for this shell. +set -a +source .env.test +set +a + +echo "Starting isolated test databases..." +docker compose -f docker-compose.test.yml up -d + +echo "Running second (test) bot with .env.test configuration..." +uv run python main.py From bfb7d3f6c3de921e2a3cddfebcd1f8aa1e50cfca Mon Sep 17 00:00:00 2001 From: Karissa Date: Mon, 6 Jul 2026 13:39:48 -0700 Subject: [PATCH 3/3] feat: add feedback logging, GitHub citations, and UX improvements Add thumbs up/down reaction logging via on_raw_reaction_add handler. Add bot_reply_message_id column to discord_interaction_logs. Build clickable GitHub citation links from source document metadata. Preserve file_path in wiki doc metadata during ingestion. Add Gathering information placeholder message while bot processes. Fix docker-compose.test.yml port to 15432 for Windows compatibility. Update prompts.py with improved QA and condense question templates. Co-authored-by: Cursor --- systems/docgpt/.env.example | 12 --- systems/docgpt/.env.test.example | 8 +- systems/docgpt/docker-compose.test.yml | 2 +- .../docgpt/src/adapters/content/git/wiki.py | 4 + systems/docgpt/src/app/discord.py | 87 ++++++++++++++++++- systems/docgpt/src/logging/discord_logger.py | 26 ++++-- 6 files changed, 118 insertions(+), 21 deletions(-) delete mode 100644 systems/docgpt/.env.example diff --git a/systems/docgpt/.env.example b/systems/docgpt/.env.example deleted file mode 100644 index 7d77f54..0000000 --- a/systems/docgpt/.env.example +++ /dev/null @@ -1,12 +0,0 @@ -# Gemini API key (or set GOOGLE_API_KEY) -AI_GEMINI_APIKEY= - -# Discord bot token -APP_DISCORD_TOKEN= - -# PostgreSQL for vector storage (must match docker-compose or your Postgres) -# Format: postgresql+psycopg://USER:PASSWORD@HOST:PORT/DATABASE -STORAGE_VECTOR_URL=postgresql+psycopg://root:example@localhost:5432/postgres - -# MongoDB for chat memory -STORAGE_MEMORY_URL=mongodb://root:example@localhost:27017 diff --git a/systems/docgpt/.env.test.example b/systems/docgpt/.env.test.example index 360e6b0..63f92c0 100644 --- a/systems/docgpt/.env.test.example +++ b/systems/docgpt/.env.test.example @@ -1,6 +1,12 @@ # Gemini API key (or set GOOGLE_API_KEY) AI_GEMINI_APIKEY= +# Gemini model (1.5-flash was retired; use a current model) +AI_GEMINI_MODEL=gemini-2.0-flash + +# Gemini model (1.5-flash was retired; use a current model) +AI_GEMINI_MODEL=gemini-2.0-flash + # Discord bot token (use a dedicated test bot token) APP_DISCORD_TOKEN= @@ -12,7 +18,7 @@ STORAGE_VECTOR_BACKEND=pgvector STORAGE_VECTOR_URL=postgresql+psycopg://root:example@localhost:55432/docgpt_test # MongoDB test database (docker-compose.test.yml) -STORAGE_MEMORY_URL=mongodb://root:example@localhost:27018/docgpt_test +STORAGE_MEMORY_URL=mongodb://root:example@localhost:27018/docgpt_test?authSource=admin # Optional LOG_LEVEL=INFO diff --git a/systems/docgpt/docker-compose.test.yml b/systems/docgpt/docker-compose.test.yml index fe3794f..41f9e76 100644 --- a/systems/docgpt/docker-compose.test.yml +++ b/systems/docgpt/docker-compose.test.yml @@ -6,7 +6,7 @@ services: container_name: vector_storage_test restart: always ports: - - "55432:5432" + - "15432:5432" environment: - POSTGRES_USER=root - POSTGRES_PASSWORD=example diff --git a/systems/docgpt/src/adapters/content/git/wiki.py b/systems/docgpt/src/adapters/content/git/wiki.py index 4b5db5e..2429e72 100644 --- a/systems/docgpt/src/adapters/content/git/wiki.py +++ b/systems/docgpt/src/adapters/content/git/wiki.py @@ -60,6 +60,10 @@ def _get_docs(self, path: Path) -> Iterable[Document]: @validate_call def get_by_path(self, project: str, path: Path) -> Iterable[Content]: for doc in self._get_docs(path): + # Preserve the original filename for citation linking before + # Content.from_document overwrites the "source" field. + if "file_path" not in doc.metadata and "source" in doc.metadata: + doc.metadata["file_path"] = Path(doc.metadata["source"]).name yield Content.from_document( doc, source=path.name, diff --git a/systems/docgpt/src/app/discord.py b/systems/docgpt/src/app/discord.py index 547fb5e..513ab98 100644 --- a/systems/docgpt/src/app/discord.py +++ b/systems/docgpt/src/app/discord.py @@ -1,5 +1,7 @@ import asyncio import logging +import os +from pathlib import Path from typing import Any import discord @@ -138,6 +140,8 @@ async def on_message( user_message = await channel.fetch_message(message.id) message_content = user_message.clean_content + thinking_msg = await channel.send("_Gathering information..._") + # LangChain / LLM work is synchronous; run off the event loop so other # users' slash commands (e.g. /help_me defer) are not starved. result: dict[str, Any] = await asyncio.to_thread( @@ -145,7 +149,50 @@ async def on_message( message_content, session_id=str(channel.id), ) + await thinking_msg.delete() response = result["answer"] + + # Build citation block from source documents (top 3 unique sources). + _GITHUB_BASE = "https://github.com/Rdatatable/data.table" + _MAX_CITATIONS = 3 + source_docs = result.get("source_documents") or [] + seen: set[str] = set() + citation_lines: list[str] = [] + for doc in source_docs: + metadata = getattr(doc, "metadata", {}) or {} + source = metadata.get("source") or "" + file_path = metadata.get("file_path") or "" + + # Deduplicate by file_path if available, otherwise source. + # Stop after MAX_CITATIONS unique sources. + dedup_key = file_path or source + if not dedup_key or dedup_key in seen: + continue + if len(citation_lines) >= _MAX_CITATIONS: + break + seen.add(dedup_key) + + if source.startswith("http://") or source.startswith("https://"): + citation_lines.append(f"- <{source}>") + elif source.endswith(".wiki"): + # Wiki page: strip .md extension to get the page name. + if file_path: + page = Path(file_path).stem + url = f"{_GITHUB_BASE}/wiki/{page}" + citation_lines.append(f"- [**{page}**](<{url}>)") + else: + # Old ingestion record without file_path — link to wiki home. + citation_lines.append(f"- [**data.table wiki**](<{_GITHUB_BASE}/wiki>)") + elif file_path: + # Source code file: link to the file on GitHub. + url = f"{_GITHUB_BASE}/blob/master/{file_path}" + citation_lines.append(f"- [**{file_path}**](<{url}>)") + else: + citation_lines.append(f"- `{os.path.basename(source)}`") + + if citation_lines: + response = response.rstrip() + "\n\n**Sources:**\n" + "\n".join(citation_lines) + response_chunks = MarkdownTextSplitter( chunk_size=MAX_MESSAGE_LEN, chunk_overlap=0, @@ -160,13 +207,13 @@ async def on_message( if first_reply_message is None: first_reply_message = sent - # Add feedback reactions to the assistant's first reply (the answer), - # not to the original user question message. + # Add feedback reactions to the assistant's first reply. if first_reply_message is not None: try: await first_reply_message.add_reaction("👍") await first_reply_message.add_reaction("👎") except Exception: + log = logging.getLogger(__name__) log.exception("Failed to add feedback reactions to assistant reply") try: @@ -179,6 +226,7 @@ async def on_message( discord_channel_id=str(channel.id), discord_thread_id=str(channel.id), discord_message_id=str(message.id), + bot_reply_message_id=str(first_reply_message.id) if first_reply_message else None, ) except Exception as e: log = logging.getLogger(__name__) @@ -198,3 +246,38 @@ async def on_message( log.warning("generate_title returned empty/unusable title: %r", title) except Exception as e: log.error("Failed to generate thread title: %s", e) + + +@BOT.event +@inject +async def on_raw_reaction_add( + payload: discord.RawReactionActionEvent, + *, + interaction_logger: DiscordInteractionLogger = Provide[ + Settings.logging.discord_logger + ], +) -> None: + log = logging.getLogger(__name__) + + # Ignore the bot's own reactions. + if BOT.user and payload.user_id == BOT.user.id: + return + + emoji = str(payload.emoji) + if emoji not in ("👍", "👎"): + return + + thumbs_up = emoji == "👍" + + try: + updated = await asyncio.to_thread( + interaction_logger.log_feedback, + bot_reply_message_id=str(payload.message_id), + thumbs_up=thumbs_up, + ) + if updated: + log.debug("Feedback logged: %s for message %s", emoji, payload.message_id) + else: + log.debug("Reaction %s on message %s matched no log row", emoji, payload.message_id) + except Exception as e: + log.error("Failed to log feedback reaction: %s", e) diff --git a/systems/docgpt/src/logging/discord_logger.py b/systems/docgpt/src/logging/discord_logger.py index dec15c9..787044f 100644 --- a/systems/docgpt/src/logging/discord_logger.py +++ b/systems/docgpt/src/logging/discord_logger.py @@ -20,6 +20,7 @@ class DiscordInteractionLogEntry: discord_channel_id: Optional[str] = None discord_thread_id: Optional[str] = None discord_message_id: Optional[str] = None + bot_reply_message_id: Optional[str] = None candidate_a_answer: Optional[str] = None candidate_b_answer: Optional[str] = None feedback_selected_candidate: Optional[str] = None @@ -52,6 +53,7 @@ def _ensure_schema(self) -> None: discord_channel_id TEXT NULL, discord_thread_id TEXT NULL, discord_message_id TEXT NULL, + bot_reply_message_id TEXT NULL, question TEXT NOT NULL, rag_answer TEXT NULL, rag_context TEXT NULL, @@ -73,6 +75,16 @@ def _ensure_schema(self) -> None: cur.execute( "CREATE INDEX IF NOT EXISTS idx_discord_logs_user_id ON discord_interaction_logs(discord_user_id);" ) + # Add bot_reply_message_id column if it doesn't exist yet (migration). + cur.execute( + """ + ALTER TABLE discord_interaction_logs + ADD COLUMN IF NOT EXISTS bot_reply_message_id TEXT NULL; + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_discord_logs_bot_reply_msg ON discord_interaction_logs(bot_reply_message_id);" + ) conn.commit() def log_interaction( @@ -86,6 +98,7 @@ def log_interaction( discord_channel_id: Optional[str], discord_thread_id: Optional[str], discord_message_id: Optional[str], + bot_reply_message_id: Optional[str] = None, candidate_a_answer: Optional[str] = None, candidate_b_answer: Optional[str] = None, ) -> int: @@ -103,10 +116,11 @@ def log_interaction( discord_channel_id, discord_thread_id, discord_message_id, + bot_reply_message_id, candidate_a_answer, candidate_b_answer ) VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s ) RETURNING id; """, @@ -120,6 +134,7 @@ def log_interaction( discord_channel_id, discord_thread_id, discord_message_id, + bot_reply_message_id, candidate_a_answer, candidate_b_answer, ), @@ -138,6 +153,7 @@ def log_entry(self, entry: DiscordInteractionLogEntry) -> int: discord_channel_id=entry.discord_channel_id, discord_thread_id=entry.discord_thread_id, discord_message_id=entry.discord_message_id, + bot_reply_message_id=entry.bot_reply_message_id, candidate_a_answer=entry.candidate_a_answer, candidate_b_answer=entry.candidate_b_answer, ) @@ -145,10 +161,10 @@ def log_entry(self, entry: DiscordInteractionLogEntry) -> int: def log_feedback( self, *, - discord_message_id: str, + bot_reply_message_id: str, thumbs_up: bool | None, ) -> bool: - """Update feedback fields for an existing interaction log row. + """Update feedback fields for a log row matched by the bot's reply message ID. Returns True if a row was updated, otherwise False. """ @@ -161,14 +177,14 @@ def log_feedback( WHEN %s IS NULL THEN NULL ELSE NOW() END - WHERE discord_message_id = %s + WHERE bot_reply_message_id = %s AND rag_name = %s RETURNING id; """, ( thumbs_up, thumbs_up, - discord_message_id, + bot_reply_message_id, self._rag_name, ), )