From 7143cb33d4e0c01faddeb8388b9f98ec44925ef6 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Sun, 31 Aug 2025 00:27:14 +0300 Subject: [PATCH 1/2] add steps in db load --- api/routes/database.py | 125 +++++++++++++-------- app/public/css/modals.css | 37 ++++++ app/templates/components/database_modal.j2 | 6 + app/ts/modules/modals.ts | 118 ++++++++++++++++--- 4 files changed, 222 insertions(+), 64 deletions(-) diff --git a/api/routes/database.py b/api/routes/database.py index d6a17547..6f18c4ac 100644 --- a/api/routes/database.py +++ b/api/routes/database.py @@ -1,8 +1,11 @@ """Database connection routes for the text2sql API.""" + import logging +import json +import time from fastapi import APIRouter, Request, HTTPException -from fastapi.responses import JSONResponse +from fastapi.responses import StreamingResponse from pydantic import BaseModel from api.auth.user_management import token_required @@ -11,6 +14,8 @@ database_router = APIRouter() +# Use the same delimiter as in the JavaScript frontend for streaming chunks +MESSAGE_DELIMITER = "|||FALKORDB_MESSAGE_BOUNDARY|||" class DatabaseConnectionRequest(BaseModel): """Database connection request model. @@ -18,8 +23,8 @@ class DatabaseConnectionRequest(BaseModel): Args: BaseModel (_type_): _description_ """ - url: str + url: str @database_router.post("/database", operation_id="connect_database") @token_required @@ -27,7 +32,7 @@ async def connect_database(request: Request, db_request: DatabaseConnectionReque """ Accepts a JSON payload with a database URL and attempts to connect. Supports both PostgreSQL and MySQL databases. - Returns success or error message. + Streams progress steps as a sequence of JSON messages separated by MESSAGE_DELIMITER. """ url = db_request.url if not url: @@ -37,52 +42,78 @@ async def connect_database(request: Request, db_request: DatabaseConnectionReque if not isinstance(url, str) or len(url.strip()) == 0: raise HTTPException(status_code=400, detail="Invalid URL format") - try: - success = False - result = "" - - # Check for PostgreSQL URL - if url.startswith("postgres://") or url.startswith("postgresql://"): + async def generate(): + overall_start = time.perf_counter() + try: + # Step 1: Start + yield json.dumps( + { + "type": "reasoning_step", + "message": "Step 1: Starting database connection", + } + ) + MESSAGE_DELIMITER + + # Step 2: Determine type + db_type = None + if url.startswith("postgres://") or url.startswith("postgresql://"): + db_type = "postgresql" + loader = PostgresLoader + elif url.startswith("mysql://"): + db_type = "mysql" + loader = MySQLLoader + else: + yield json.dumps( + {"type": "error", "message": "Invalid database URL format"} + ) + MESSAGE_DELIMITER + return + + yield json.dumps( + { + "type": "reasoning_step", + "message": f"Step 2: Detected database type: {db_type}. " + "Attempting to load schema...", + } + ) + MESSAGE_DELIMITER + + # Step 3: Attempt to load schema using the loader try: - # Attempt to connect/load using the PostgreSQL loader - success, result = await PostgresLoader.load(request.state.user_id, url) - except (ValueError, ConnectionError) as e: - logging.error("PostgreSQL connection error: %s", str(e)) - raise HTTPException( - status_code=500, - detail="Failed to connect to PostgreSQL database", + load_start = time.perf_counter() + success, result = await loader.load(request.state.user_id, url) + load_elapsed = time.perf_counter() - load_start + logging.info( + "Database load attempt finished in %.2f seconds", load_elapsed ) - # Check for MySQL URL - elif url.startswith("mysql://"): - try: - # Attempt to connect/load using the MySQL loader - success, result = await MySQLLoader.load(request.state.user_id, url) - except (ValueError, ConnectionError) as e: - logging.error("MySQL connection error: %s", str(e)) - raise HTTPException( - status_code=500, detail="Failed to connect to MySQL database" - ) - - else: - raise HTTPException( - status_code=400, - detail=( - "Invalid database URL. Supported formats: postgresql:// " - "or mysql://" - ), + if success: + yield json.dumps( + { + "type": "final_result", + "success": True, + "message": "Database connected and schema loaded successfully", + } + ) + MESSAGE_DELIMITER + else: + # Don't stream the full internal result; give higher-level error + logging.error("Database loader failed: %s", str(result)) + yield json.dumps( + {"type": "error", "message": "Failed to load database schema"} + ) + MESSAGE_DELIMITER + except Exception as e: + logging.exception("Error while loading database schema: %s", str(e)) + yield json.dumps( + {"type": "error", "message": "Error connecting to database"} + ) + MESSAGE_DELIMITER + + except Exception as e: + logging.exception("Unexpected error in connect_database stream: %s", str(e)) + yield json.dumps( + {"type": "error", "message": "Internal server error"} + ) + MESSAGE_DELIMITER + finally: + overall_elapsed = time.perf_counter() - overall_start + logging.info( + "connect_database processing completed - Total time: %.2f seconds", + overall_elapsed, ) - if success: - return JSONResponse(content={ - "success": True, - "message": "Database connected successfully" - }) - - # Don't return detailed error messages to prevent information exposure - logging.error("Database loader failed: %s", result) - raise HTTPException(status_code=400, detail="Failed to load database schema") - - except (ValueError, TypeError) as e: - logging.error("Unexpected error in database connection: %s", str(e)) - raise HTTPException(status_code=500, detail="Internal server error") + return StreamingResponse(generate(), media_type="application/json") diff --git a/app/public/css/modals.css b/app/public/css/modals.css index ad438fb1..7abcd2e4 100644 --- a/app/public/css/modals.css +++ b/app/public/css/modals.css @@ -399,6 +399,43 @@ overflow-x: auto; } +/* Styles for incremental database connection steps shown in the connect modal */ +#db-connection-steps { + margin: 16px 24px; + font-size: 14px; + color: var(--text-primary); +} + +#db-connection-steps-list { + list-style: none; + padding: 0; + margin: 0; + max-height: 220px; + overflow: auto; +} + +.db-connection-step { + display: flex; + align-items: center; + margin: 8px 0; +} + +.db-connection-step .step-icon { + display: inline-flex; + width: 20px; + height: 20px; + margin-right: 8px; + border-radius: 50%; + align-items: center; + justify-content: center; + font-size: 12px; + line-height: 20px; +} + +.db-connection-step .step-icon.pending { color: #1f6feb; } +.db-connection-step .step-icon.success { color: #16a34a; } +.db-connection-step .step-icon.error { color: #dc2626; } + .alert { padding: 1.5em; border-radius: 6px; diff --git a/app/templates/components/database_modal.j2 b/app/templates/components/database_modal.j2 index 7795401a..c2580112 100644 --- a/app/templates/components/database_modal.j2 +++ b/app/templates/components/database_modal.j2 @@ -3,6 +3,12 @@

Connect to Database

+ + +
+
    +
    +