Added new features

systems/docgpt/.env.test.example

@@ -0,0 +1,24 @@
+# Gemini API key (or set GOOGLE_API_KEY)
+AI_GEMINI_APIKEY=
+
+# Gemini model (1.5-flash was retired; use a current model)
+AI_GEMINI_MODEL=gemini-2.0-flash
+
+# Gemini model (1.5-flash was retired; use a current model)
+AI_GEMINI_MODEL=gemini-2.0-flash
+
+# Discord bot token (use a dedicated test bot token)
+APP_DISCORD_TOKEN=
+
+# Force pgvector backend for isolated test database runs
+STORAGE_VECTOR_BACKEND=pgvector
+
+# PostgreSQL test database (docker-compose.test.yml)
+# Format: postgresql+psycopg://USER:PASSWORD@HOST:PORT/DATABASE
+STORAGE_VECTOR_URL=postgresql+psycopg://root:example@localhost:55432/docgpt_test
+
+# MongoDB test database (docker-compose.test.yml)
+STORAGE_MEMORY_URL=mongodb://root:example@localhost:27018/docgpt_test?authSource=admin
+
+# Optional
+LOG_LEVEL=INFO

systems/docgpt/README.md

@@ -52,3 +52,70 @@ Ask about the R data.table package documentation and contribution guide.
 2. Run ```docker compose up```
 3. Ingest data (once): ```uv run python main.py --ingest```
 4. Start the Discord bot: ```uv run python main.py```
+
+## Manual bot testing with separate test DB
+
+Use this flow when you want to interact with the bot in Discord without touching normal dev data.
+
+1. Create a test env file from the template:
+   ```powershell
+   Copy-Item .env.test.example .env.test
+   ```
+2. Fill in `AI_GEMINI_APIKEY` and `APP_DISCORD_TOKEN` in `.env.test`.
+3. Start isolated test databases:
+   ```powershell
+   docker compose -f docker-compose.test.yml up -d
+   ```
+4. Load `.env.test` into the current PowerShell session:
+   ```powershell
+   Get-Content .env.test | ForEach-Object {
+     if ($_ -match '^\s*#' -or $_ -match '^\s*$') { return }
+     $name, $value = $_ -split '=', 2
+     Set-Item -Path "Env:$name" -Value $value
+   }
+   ```
+5. Ingest documents into the test vector database:
+   ```powershell
+   uv run python main.py --ingest
+   ```
+6. Run the Discord bot using the test DB settings:
+   ```powershell
+   uv run python main.py
+   ```
+7. Tear down and wipe test data when done:
+   ```powershell
+   docker compose -f docker-compose.test.yml down -v
+   ```
+
+This keeps vector data and chat memory isolated to test services (`localhost:55432`, `localhost:27018`) and removes persisted test data on teardown.
+
+## Run a second bot on EC2 (side-by-side with prod)
+
+Use this when your production bot is already running and you want a separate test bot process.
+
+1. Create a second Discord bot application/token (test-only) and add it to a test server.
+2. Prepare test environment values:
+   ```bash
+   cp .env.test.example .env.test
+   ```
+3. Edit `.env.test` and set:
+   - `APP_DISCORD_TOKEN` to the test bot token
+   - `AI_GEMINI_APIKEY`
+4. Ingest test data once:
+   ```bash
+   set -a && source .env.test && set +a
+   docker compose -f docker-compose.test.yml up -d
+   uv run python main.py --ingest
+   ```
+5. Run the second bot:
+   ```bash
+   ./run-test-bot.sh
+   ```
+
+Notes:
+- Do not reuse the production bot token for the test bot.
+- Production bot keeps using `.env`; test bot uses `.env.test`.
+- Stop and wipe test data when finished:
+  ```bash
+  docker compose -f docker-compose.test.yml down -v
+  ```

systems/docgpt/docker-compose.test.yml

@@ -0,0 +1,31 @@
+version: "3"
+
+services:
+  vector_storage_test:
+    image: ankane/pgvector
+    container_name: vector_storage_test
+    restart: always
+    ports:
+      - "15432:5432"
+    environment:
+      - POSTGRES_USER=root
+      - POSTGRES_PASSWORD=example
+      - POSTGRES_DB=docgpt_test
+    volumes:
+      - docgpt_test_postgres_data:/var/lib/postgresql/data
+
+  memory_storage_test:
+    image: mongo
+    container_name: memory_storage_test
+    restart: always
+    ports:
+      - "27018:27017"
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+    volumes:
+      - docgpt_test_mongo_data:/data/db
+
+volumes:
+  docgpt_test_postgres_data:
+  docgpt_test_mongo_data:

systems/docgpt/run-test-bot.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ ! -f ".env.test" ]]; then
+  echo ".env.test not found. Create it from .env.test.example first."
+  exit 1
+fi
+
+# Export all variables from .env.test for this shell.
+set -a
+source .env.test
+set +a
+
+echo "Starting isolated test databases..."
+docker compose -f docker-compose.test.yml up -d
+
+echo "Running second (test) bot with .env.test configuration..."
+uv run python main.py

systems/docgpt/src/adapters/content/git/wiki.py

@@ -60,6 +60,10 @@ def _get_docs(self, path: Path) -> Iterable[Document]:
     @validate_call
     def get_by_path(self, project: str, path: Path) -> Iterable[Content]:
         for doc in self._get_docs(path):
+            # Preserve the original filename for citation linking before
+            # Content.from_document overwrites the "source" field.
+            if "file_path" not in doc.metadata and "source" in doc.metadata:
+                doc.metadata["file_path"] = Path(doc.metadata["source"]).name
             yield Content.from_document(
                 doc,
                 source=path.name,

systems/docgpt/src/app/discord.py

@@ -1,5 +1,7 @@
 import asyncio
 import logging
+import os
+from pathlib import Path
 from typing import Any
 
 import discord
@@ -138,14 +140,59 @@ async def on_message(
     user_message = await channel.fetch_message(message.id)
     message_content = user_message.clean_content
 
+    thinking_msg = await channel.send("_Gathering information..._")
+
     # LangChain / LLM work is synchronous; run off the event loop so other
     # users' slash commands (e.g. /help_me defer) are not starved.
     result: dict[str, Any] = await asyncio.to_thread(
         assistant.prompt_with_metadata,
         message_content,
         session_id=str(channel.id),
     )
+    await thinking_msg.delete()
     response = result["answer"]
+
+    # Build citation block from source documents (top 3 unique sources).
+    _GITHUB_BASE = "https://github.com/Rdatatable/data.table"
+    _MAX_CITATIONS = 3
+    source_docs = result.get("source_documents") or []
+    seen: set[str] = set()
+    citation_lines: list[str] = []
+    for doc in source_docs:
+        metadata = getattr(doc, "metadata", {}) or {}
+        source = metadata.get("source") or ""
+        file_path = metadata.get("file_path") or ""
+
+        # Deduplicate by file_path if available, otherwise source.
+        # Stop after MAX_CITATIONS unique sources.
+        dedup_key = file_path or source
+        if not dedup_key or dedup_key in seen:
+            continue
+        if len(citation_lines) >= _MAX_CITATIONS:
+            break
+        seen.add(dedup_key)
+
+        if source.startswith("http://") or source.startswith("https://"):
+            citation_lines.append(f"- <{source}>")
+        elif source.endswith(".wiki"):
+            # Wiki page: strip .md extension to get the page name.
+            if file_path:
+                page = Path(file_path).stem
+                url = f"{_GITHUB_BASE}/wiki/{page}"
+                citation_lines.append(f"- [**{page}**](<{url}>)")
+            else:
+                # Old ingestion record without file_path — link to wiki home.
+                citation_lines.append(f"- [**data.table wiki**](<{_GITHUB_BASE}/wiki>)")
+        elif file_path:
+            # Source code file: link to the file on GitHub.
+            url = f"{_GITHUB_BASE}/blob/master/{file_path}"
+            citation_lines.append(f"- [**{file_path}**](<{url}>)")
+        else:
+            citation_lines.append(f"- `{os.path.basename(source)}`")
+
+    if citation_lines:
+        response = response.rstrip() + "\n\n**Sources:**\n" + "\n".join(citation_lines)
+
     response_chunks = MarkdownTextSplitter(
         chunk_size=MAX_MESSAGE_LEN,
         chunk_overlap=0,
@@ -154,8 +201,20 @@ async def on_message(
         add_start_index=True,
     ).split_text(response)
 
+    first_reply_message: discord.Message | None = None
     for reply in response_chunks:
-        await user_message.reply(reply)
+        sent = await user_message.reply(reply)
+        if first_reply_message is None:
+            first_reply_message = sent
+
+    # Add feedback reactions to the assistant's first reply.
+    if first_reply_message is not None:
+        try:
+            await first_reply_message.add_reaction("👍")
+            await first_reply_message.add_reaction("👎")
+        except Exception:
+            log = logging.getLogger(__name__)
+            log.exception("Failed to add feedback reactions to assistant reply")
 
     try:
         interaction_logger.log_interaction(
@@ -167,6 +226,7 @@ async def on_message(
             discord_channel_id=str(channel.id),
             discord_thread_id=str(channel.id),
             discord_message_id=str(message.id),
+            bot_reply_message_id=str(first_reply_message.id) if first_reply_message else None,
         )
     except Exception as e:
         log = logging.getLogger(__name__)
@@ -186,3 +246,38 @@ async def on_message(
                 log.warning("generate_title returned empty/unusable title: %r", title)
         except Exception as e:
             log.error("Failed to generate thread title: %s", e)
+
+
+@BOT.event
+@inject
+async def on_raw_reaction_add(
+    payload: discord.RawReactionActionEvent,
+    *,
+    interaction_logger: DiscordInteractionLogger = Provide[
+        Settings.logging.discord_logger
+    ],
+) -> None:
+    log = logging.getLogger(__name__)
+
+    # Ignore the bot's own reactions.
+    if BOT.user and payload.user_id == BOT.user.id:
+        return
+
+    emoji = str(payload.emoji)
+    if emoji not in ("👍", "👎"):
+        return
+
+    thumbs_up = emoji == "👍"
+
+    try:
+        updated = await asyncio.to_thread(
+            interaction_logger.log_feedback,
+            bot_reply_message_id=str(payload.message_id),
+            thumbs_up=thumbs_up,
+        )
+        if updated:
+            log.debug("Feedback logged: %s for message %s", emoji, payload.message_id)
+        else:
+            log.debug("Reaction %s on message %s matched no log row", emoji, payload.message_id)
+    except Exception as e:
+        log.error("Failed to log feedback reaction: %s", e)