diff --git a/README.md b/README.md index d5b905c0..e78be260 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Run a local demo instance in seconds. ```bash # Clone and launch instantly +cp .env.example .env docker compose -f docker-compose.yml -f docker-compose.demo.yml up --build ``` diff --git a/backend-services/doorman.py b/backend-services/doorman.py index e423923e..890974be 100755 --- a/backend-services/doorman.py +++ b/backend-services/doorman.py @@ -96,6 +96,7 @@ from routes.grpc_routes import grpc_router from utils.audit_util import audit from middleware.security_audit_middleware import SecurityAuditMiddleware +from middleware.logging_middleware import GlobalLoggingMiddleware from middleware.latency_injection_middleware import LatencyInjectionMiddleware from middleware.websocket_reject_middleware import WebSocketRejectMiddleware from utils.auth_blacklist import purge_expired_tokens @@ -111,6 +112,7 @@ restore_memory_from_file, ) from utils.metrics_util import metrics_store +from utils.enhanced_metrics_util import enhanced_metrics_store from utils.response_util import process_response from utils.security_settings_util import ( get_cached_settings, @@ -368,11 +370,18 @@ async def app_lifespan(app: FastAPI): except Exception as e: gateway_logger.debug(f'Metrics restore skipped: {e}') + ENHANCED_METRICS_FILE = os.path.join(LOGS_DIR, 'enhanced_metrics.json') + try: + enhanced_metrics_store.load_from_file(ENHANCED_METRICS_FILE) + except Exception as e: + gateway_logger.debug(f'Enhanced metrics restore skipped: {e}') + async def _metrics_autosave(interval_s: int = 60): while True: try: await asyncio.sleep(interval_s) metrics_store.save_to_file(METRICS_FILE) + enhanced_metrics_store.save_to_file(ENHANCED_METRICS_FILE) except asyncio.CancelledError: break except Exception: @@ -647,6 +656,8 @@ async def _sighup_reload(): try: METRICS_FILE = os.path.join(LOGS_DIR, 'metrics.json') metrics_store.save_to_file(METRICS_FILE) + ENHANCED_METRICS_FILE = os.path.join(LOGS_DIR, 'enhanced_metrics.json') + enhanced_metrics_store.save_to_file(ENHANCED_METRICS_FILE) except Exception: pass @@ -1484,6 +1495,7 @@ async def send_wrapper(message): doorman.add_middleware(PlatformCORSMiddleware) +doorman.add_middleware(GlobalLoggingMiddleware) # Add tier-based rate limiting middleware (skip in live/test to avoid 429 floods) try: @@ -1514,41 +1526,45 @@ async def send_wrapper(message): @doorman.middleware('http') async def request_id_middleware(request: Request, call_next): try: + from utils.correlation_util import get_correlation_id, set_correlation_id + rid = ( - request.headers.get('x-request-id') + getattr(request.state, 'request_id', None) + or get_correlation_id() + or request.headers.get('x-request-id') or request.headers.get('request-id') - or request.headers.get('x-request-id'.title()) + or request.headers.get('X-Request-ID') ) if not rid: rid = str(uuid.uuid4()) - try: + # Store in state + if not hasattr(request.state, 'request_id'): request.state.request_id = rid - except Exception: - pass - try: - from utils.correlation_util import set_correlation_id + # Ensure correlation ID is set + set_correlation_id(rid) - set_correlation_id(rid) - except Exception: - pass + # Optional logging try: settings = get_cached_settings() - trust_xff = bool(settings.get('trust_x_forwarded_for')) + trust_xff = bool(settings.get('trust_x_forwarded_for', False)) direct_ip = getattr(getattr(request, 'client', None), 'host', None) effective_ip = _policy_get_client_ip(request, trust_xff) gateway_logger.info( - f'{rid} | Entry: client_ip={direct_ip} effective_ip={effective_ip} method={request.method} path={str(request.url.path)}' + f'Entry: client_ip={direct_ip} effective_ip={effective_ip} method={request.method} path={str(request.url.path)}' ) except Exception: pass + response = await call_next(request) + try: response.headers['X-Request-ID'] = rid response.headers['request_id'] = rid except Exception as e: gateway_logger.warning(f'Failed to set response headers: {str(e)}') + return response except Exception as e: gateway_logger.error(f'Request ID middleware error: {str(e)}', exc_info=True) @@ -1629,10 +1645,12 @@ async def security_headers(request: Request, call_next): # Build formatters class JSONFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: + request_id = getattr(record, 'request_id', 'no-request-id') payload = { 'time': self.formatTime(record, '%Y-%m-%dT%H:%M:%S'), 'name': record.name, 'level': record.levelname, + 'request_id': request_id, 'message': record.getMessage(), } try: @@ -1654,7 +1672,7 @@ def format(self, record: logging.LogRecord) -> str: _file_handler.setFormatter( JSONFormatter() if _fmt_is_json - else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(request_id)s | %(message)s') ) except Exception as _e: # Attempt fallback to /tmp which is commonly writable on containers @@ -1670,7 +1688,7 @@ def format(self, record: logging.LogRecord) -> str: _file_handler.setFormatter( JSONFormatter() if _fmt_is_json - else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(request_id)s | %(message)s') ) logging.getLogger('doorman.gateway').warning( f'Primary LOGS_DIR={LOGS_DIR} not writable ({_e}); falling back to {_tmp_dir}' @@ -1685,12 +1703,16 @@ def format(self, record: logging.LogRecord) -> str: # Configure all doorman loggers to use the same handler and prevent propagation def configure_logger(logger_name: str) -> logging.Logger: logger = logging.getLogger(logger_name) - logger.setLevel(logging.INFO) + log_level = os.getenv('LOG_LEVEL', 'INFO').upper() + numeric_level = getattr(logging, log_level, logging.INFO) + logger.setLevel(numeric_level) logger.propagate = False for handler in logger.handlers[:]: logger.removeHandler(handler) + from utils.correlation_util import correlation_id, RequestIdFilter + class RedactFilter(logging.Filter): """Comprehensive logging redaction filter for sensitive data. @@ -1767,12 +1789,13 @@ def filter(self, record: logging.LogRecord) -> bool: return True console = logging.StreamHandler(stream=sys.stdout) - console.setLevel(logging.INFO) + console.setLevel(numeric_level) console.setFormatter( JSONFormatter() if _fmt_is_json - else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + else logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(request_id)s | %(message)s') ) + console.addFilter(RequestIdFilter()) console.addFilter(RedactFilter()) logger.addHandler(console) @@ -1780,6 +1803,7 @@ def filter(self, record: logging.LogRecord) -> bool: if not any( isinstance(f, logging.Filter) and hasattr(f, 'PATTERNS') for f in _file_handler.filters ): + _file_handler.addFilter(RequestIdFilter()) _file_handler.addFilter(RedactFilter()) logger.addHandler(_file_handler) return logger @@ -1795,7 +1819,9 @@ def filter(self, record: logging.LogRecord) -> bool: _mem_enabled = os.getenv('MEMORY_LOG_ENABLED', 'true').lower() != 'false' if _mem_enabled: - _mem_handler = MemoryLogHandler() + _lvl = os.getenv('LOG_LEVEL', 'INFO').upper() + _num_lvl = getattr(logging, _lvl, logging.INFO) + _mem_handler = MemoryLogHandler(level=_num_lvl) for _name in ( 'doorman.gateway', 'doorman.logging', diff --git a/backend-services/middleware/analytics_middleware.py b/backend-services/middleware/analytics_middleware.py index bb48183a..11a975c5 100644 --- a/backend-services/middleware/analytics_middleware.py +++ b/backend-services/middleware/analytics_middleware.py @@ -48,14 +48,27 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: method = request.method path = str(request.url.path) - # Estimate request size (headers + body) + # Estimate request size (headers + body + protocol overhead) request_size = 0 try: - # Headers size - request_size += sum(len(k) + len(v) for k, v in request.headers.items()) - # Body size (if available) + # Request line: "METHOD /path?query HTTP/1.1\r\n" + request_line = f"{request.method} {request.url.path}" + if request.url.query: + request_line += f"?{request.url.query}" + request_line += " HTTP/1.1\r\n" + request_size += len(request_line) + + # Headers with proper separators (key: value\r\n) + for k, v in request.headers.items(): + request_size += len(f"{k}: {v}\r\n") + request_size += 2 # Final \r\n separator + + # Body size if 'content-length' in request.headers: request_size += int(request.headers['content-length']) + elif hasattr(request, '_body') and request._body: + # Fallback: use actual body if available + request_size += len(request._body) except Exception: pass @@ -83,14 +96,29 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: # Extract response metadata status_code = response.status_code - # Estimate response size + # Estimate response size (headers + body + protocol overhead) response_size = 0 try: - # Headers size - response_size += sum(len(k) + len(v) for k, v in response.headers.items()) - # Body size (if available) + # Status line: "HTTP/1.1 200 OK\r\n" + status_text = response.status_code + try: + from http import HTTPStatus + status_text = f"{response.status_code} {HTTPStatus(response.status_code).phrase}" + except Exception: + status_text = str(response.status_code) + response_size += len(f"HTTP/1.1 {status_text}\r\n") + + # Headers with proper separators (key: value\r\n) + for k, v in response.headers.items(): + response_size += len(f"{k}: {v}\r\n") + response_size += 2 # Final \r\n separator + + # Body size if 'content-length' in response.headers: response_size += int(response.headers['content-length']) + # Note: For streaming/chunked responses without Content-Length, + # we can't accurately measure body size in middleware without wrapping. + # This is acceptable as most responses have Content-Length. except Exception: pass @@ -104,6 +132,9 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: except Exception: pass + # Extract request_id for logging + request_id = getattr(request.state, 'request_id', None) or request.headers.get('X-Request-ID') or 'unknown' + # Parse API and endpoint from path api_key, endpoint_uri = self._parse_api_endpoint(path) diff --git a/backend-services/middleware/logging_middleware.py b/backend-services/middleware/logging_middleware.py new file mode 100644 index 00000000..31a002de --- /dev/null +++ b/backend-services/middleware/logging_middleware.py @@ -0,0 +1,68 @@ +import logging +import time +import uuid +from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import Request + +logger = logging.getLogger('doorman.gateway') + +class GlobalLoggingMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + # 1. Generate or extract Request ID + from utils.correlation_util import correlation_id + + request_id = ( + getattr(request.state, 'request_id', None) + or correlation_id.get() + or request.headers.get('X-Request-ID') + or request.headers.get('request-id') + ) + if not request_id: + request_id = str(uuid.uuid4()) + + correlation_id.set(request_id) + + # Store in state if not already present + if not hasattr(request.state, 'request_id'): + request.state.request_id = request_id + + # 2. Start Timer + start_time = time.time() + + # 3. Log Request Entry (Optional - helpful for incomplete requests) + # We can default to 'platform' or 'gateway' in type inference if needed. + # Using a consistent prefix helps. + # logger.info(f"Request started: {request.method} {request.url.path}") + + try: + # 4. Process Request + response = await call_next(request) + + # 5. Calculate Duration + duration = (time.time() - start_time) * 1000 + + # 6. Log Response + # Format matches LoggingService extraction regexes: + # Endpoint: {method} {path} + # status_code: {code} + # Total time: {ms}ms + logger.info( + f"Endpoint: {request.method} {request.url.path} " + f"| status_code: {response.status_code} " + f"| Total time: {duration:.2f}ms" + ) + + # Ensure Request ID is returned in headers for debugging + response.headers['X-Request-ID'] = request_id + + return response + + except Exception as e: + # 7. Log Unhandled Exceptions + duration = (time.time() - start_time) * 1000 + logger.error( + f"Request failed: {request.method} {request.url.path} " + f"| Error: {str(e)} | Time: {duration:.2f}ms", + exc_info=True + ) + raise diff --git a/backend-services/models/analytics_models.py b/backend-services/models/analytics_models.py index ffb53db1..f70d9e5b 100644 --- a/backend-services/models/analytics_models.py +++ b/backend-services/models/analytics_models.py @@ -131,6 +131,21 @@ def to_dict(self) -> dict: 'status_counts': dict(self.status_counts), } + @staticmethod + def from_dict(d: dict) -> EndpointMetrics: + em = EndpointMetrics( + endpoint_uri=str(d.get('endpoint_uri', '')), + method=str(d.get('method', '')), + count=int(d.get('count', 0)), + error_count=int(d.get('error_count', 0)), + # Approximate reconstruction of total_ms from average if latency list is lost/truncated + total_ms=float(d.get('avg_ms', 0.0)) * int(d.get('count', 0)), + ) + em.status_counts = {int(k): int(v) for k, v in (d.get('status_counts') or {}).items()} + # Note: We don't persist full latency list for endpoints to save space, + # so we accept starting with empty latencies. New requests will populate it. + return em + @dataclass class EnhancedMinuteBucket: @@ -262,15 +277,50 @@ def to_dict(self) -> dict: # NEW: Enhanced fields 'percentiles': percentiles.to_dict(), 'unique_users': self.get_unique_user_count(), + # Persist unique users as list + 'unique_users_list': list(self.unique_users), 'endpoint_metrics': {k: v.to_dict() for k, v in self.endpoint_metrics.items()}, - 'avg_request_size': sum(self.request_sizes) / len(self.request_sizes) - if self.request_sizes - else 0, 'avg_response_size': sum(self.response_sizes) / len(self.response_sizes) if self.response_sizes else 0, } + @staticmethod + def from_dict(d: dict) -> EnhancedMinuteBucket: + mb = EnhancedMinuteBucket( + start_ts=int(d.get('start_ts', 0)), + count=int(d.get('count', 0)), + error_count=int(d.get('error_count', 0)), + total_ms=float(d.get('total_ms', 0.0)), + bytes_in=int(d.get('bytes_in', 0)), + bytes_out=int(d.get('bytes_out', 0)), + upstream_timeouts=int(d.get('upstream_timeouts', 0)), + retries=int(d.get('retries', 0)), + ) + + # Restore dict/set fields + mb.status_counts = {int(k): int(v) for k, v in (d.get('status_counts') or {}).items()} + mb.api_counts = dict(d.get('api_counts') or {}) + mb.api_error_counts = dict(d.get('api_error_counts') or {}) + mb.user_counts = dict(d.get('user_counts') or {}) + + # Restore endpoint metrics + ep_data = d.get('endpoint_metrics') or {} + for k, v in ep_data.items(): + try: + mb.endpoint_metrics[k] = EndpointMetrics.from_dict(v) + except Exception: + pass + + # Restore unique users set + if 'unique_users_list' in d: + mb.unique_users = set(d['unique_users_list']) + elif 'user_counts' in d: + # Fallback for old data + mb.unique_users = set(d['user_counts'].keys()) + + return mb + @dataclass class AggregatedMetrics: @@ -294,6 +344,9 @@ class AggregatedMetrics: status_counts: dict[int, int] = field(default_factory=dict) api_counts: dict[str, int] = field(default_factory=dict) percentiles: PercentileMetrics | None = None + + # Store set for merging (not always persisted) + unique_users_set: set = field(default_factory=set) def to_dict(self) -> dict: return { @@ -310,8 +363,46 @@ def to_dict(self) -> dict: 'status_counts': dict(self.status_counts), 'api_counts': dict(self.api_counts), 'percentiles': self.percentiles.to_dict() if self.percentiles else None, + # Persist unique users set as list + 'unique_users_list': list(self.unique_users_set) if self.unique_users_set else [], } + @staticmethod + def from_dict(d: dict) -> AggregatedMetrics: + """Create AggregatedMetrics from dictionary.""" + am = AggregatedMetrics( + start_ts=int(d.get('start_ts', 0)), + end_ts=int(d.get('end_ts', 0)), + level=AggregationLevel(d.get('level', 'minute')), + count=int(d.get('count', 0)), + error_count=int(d.get('error_count', 0)), + total_ms=float(d.get('total_ms', 0.0)), + bytes_in=int(d.get('bytes_in', 0)), + bytes_out=int(d.get('bytes_out', 0)), + unique_users=int(d.get('unique_users', 0)), + ) + + am.status_counts = {int(k): int(v) for k, v in (d.get('status_counts') or {}).items()} + am.api_counts = dict(d.get('api_counts') or {}) + + if d.get('percentiles'): + p = d['percentiles'] + am.percentiles = PercentileMetrics( + p50=float(p.get('p50', 0)), + p75=float(p.get('p75', 0)), + p90=float(p.get('p90', 0)), + p95=float(p.get('p95', 0)), + p99=float(p.get('p99', 0)), + min=float(p.get('min', 0)), + max=float(p.get('max', 0)), + ) + + # Restore unique users set + if 'unique_users_list' in d: + am.unique_users_set = set(d['unique_users_list']) + + return am + @dataclass class AnalyticsSnapshot: diff --git a/backend-services/models/create_endpoint_model.py b/backend-services/models/create_endpoint_model.py index 0b4ccd0b..b7e24fbe 100644 --- a/backend-services/models/create_endpoint_model.py +++ b/backend-services/models/create_endpoint_model.py @@ -32,6 +32,13 @@ class CreateEndpointModel(BaseModel): description='Optional list of backend servers for this endpoint (overrides API servers)', example=['http://localhost:8082', 'http://localhost:8083'], ) + client_uri: str | None = Field( + None, + min_length=1, + max_length=255, + description='Client-facing URI (defaults to endpoint_uri if not set)', + example='/customers', + ) api_id: str | None = Field( None, description='Unique identifier for the API, auto-generated', example=None diff --git a/backend-services/models/update_endpoint_model.py b/backend-services/models/update_endpoint_model.py index c559e2be..abf351d9 100644 --- a/backend-services/models/update_endpoint_model.py +++ b/backend-services/models/update_endpoint_model.py @@ -32,6 +32,13 @@ class UpdateEndpointModel(BaseModel): description='Optional list of backend servers for this endpoint (overrides API servers)', example=['http://localhost:8082', 'http://localhost:8083'], ) + client_uri: str | None = Field( + None, + min_length=1, + max_length=255, + description='Client-facing URI (defaults to endpoint_uri if not set)', + example='/customers', + ) api_id: str | None = Field( None, min_length=1, diff --git a/backend-services/routes/analytics_routes.py b/backend-services/routes/analytics_routes.py index 25f2dfe8..7f78c5ae 100644 --- a/backend-services/routes/analytics_routes.py +++ b/backend-services/routes/analytics_routes.py @@ -109,9 +109,9 @@ async def get_analytics_overview( username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'view_analytics'): return respond_rest( @@ -169,7 +169,7 @@ async def get_analytics_overview( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -181,7 +181,7 @@ async def get_analytics_overview( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -285,7 +285,7 @@ async def get_analytics_timeseries( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -297,7 +297,7 @@ async def get_analytics_timeseries( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -372,7 +372,7 @@ async def get_top_apis( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -384,7 +384,7 @@ async def get_top_apis( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -454,7 +454,7 @@ async def get_top_users( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -466,7 +466,7 @@ async def get_top_users( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -552,7 +552,7 @@ async def get_top_endpoints( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -564,7 +564,7 @@ async def get_top_endpoints( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -668,7 +668,7 @@ async def get_api_analytics( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -680,7 +680,7 @@ async def get_api_analytics( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') # ============================================================================ @@ -767,7 +767,7 @@ async def get_user_analytics( ) except Exception as e: - logger.error(f'{request_id} | Error: {str(e)}', exc_info=True) + logger.error(f'Error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -779,4 +779,4 @@ async def get_user_analytics( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') diff --git a/backend-services/routes/api_routes.py b/backend-services/routes/api_routes.py index 2db65e81..fe2993a4 100644 --- a/backend-services/routes/api_routes.py +++ b/backend-services/routes/api_routes.py @@ -51,12 +51,12 @@ async def create_api(request: Request, api_data: CreateApiModel) -> Response: request_id = str(uuid.uuid4()) start_time = time.time() * 1000 logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') try: if not await platform_role_required_bool(username, Roles.MANAGE_APIS): - logger.warning(f'{request_id} | Permission denied for user: {username}') + logger.warning(f'Permission denied for user: {username}') return respond_rest( ResponseModel( status_code=403, @@ -77,7 +77,7 @@ async def create_api(request: Request, api_data: CreateApiModel) -> Response: ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -89,7 +89,7 @@ async def create_api(request: Request, api_data: CreateApiModel) -> Response: ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -122,9 +122,9 @@ async def update_api( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_APIS): return respond_rest( ResponseModel( @@ -146,7 +146,7 @@ async def update_api( ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -158,7 +158,7 @@ async def update_api( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -189,14 +189,14 @@ async def get_api_by_name_version(api_name: str, api_version: str, request: Requ payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest( await ApiService.get_api_by_name_version(api_name, api_version, request_id) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -208,7 +208,7 @@ async def get_api_by_name_version(api_name: str, api_version: str, request: Requ ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -239,9 +239,9 @@ async def delete_api(api_name: str, api_version: str, request: Request) -> Respo payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') result = await ApiService.delete_api(api_name, api_version, request_id) audit( request, @@ -254,7 +254,7 @@ async def delete_api(api_name: str, api_version: str, request: Request) -> Respo ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -266,7 +266,7 @@ async def delete_api(api_name: str, api_version: str, request: Request) -> Respo ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -289,9 +289,9 @@ async def get_all_apis( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await ApiService.get_apis(page, page_size, request_id)) except HTTPException as e: # Surface 401/403 properly for tests that probe unauthorized access @@ -304,7 +304,7 @@ async def get_all_apis( ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -316,7 +316,7 @@ async def get_all_apis( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @api_router.get('', description='Get all APIs (base path)', response_model=list[ApiModelResponse]) @@ -332,9 +332,9 @@ async def get_all_apis_base( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await ApiService.get_apis(page, page_size, request_id)) except HTTPException as e: return respond_rest( @@ -346,7 +346,7 @@ async def get_all_apis_base( ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, diff --git a/backend-services/routes/authorization_routes.py b/backend-services/routes/authorization_routes.py index 303ac328..fd87af05 100644 --- a/backend-services/routes/authorization_routes.py +++ b/backend-services/routes/authorization_routes.py @@ -62,8 +62,8 @@ async def authorization(request: Request): login_window = int(os.getenv('LOGIN_IP_RATE_WINDOW', '300')) rate_limit_info = await limit_by_ip(request, limit=login_limit, window=login_window) - logger.info(f'{request_id} | From: {request.client.host}:{request.client.port}') - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'From: {request.client.host}:{request.client.port}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') try: data = await request.json() except Exception: @@ -168,7 +168,7 @@ async def authorization(request: Request): if not _secure and os.getenv('ENV', '').lower() in ('production', 'prod'): logger.warning( - f'{request_id} | SECURITY WARNING: Secure cookies disabled in production environment' + f'SECURITY WARNING: Secure cookies disabled in production environment' ) _domain = os.getenv('COOKIE_DOMAIN', None) @@ -181,7 +181,7 @@ async def authorization(request: Request): _samesite = 'lax' if _samesite == 'none' and not _secure: logger.warning( - f'{request_id} | COOKIE_SAMESITE=None requires Secure cookies; downgrading to Lax for non-HTTPS' + f'COOKIE_SAMESITE=None requires Secure cookies; downgrading to Lax for non-HTTPS' ) _samesite = 'lax' @@ -272,7 +272,7 @@ async def authorization(request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -283,7 +283,7 @@ async def authorization(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -317,7 +317,7 @@ async def register(request: Request): await limit_by_ip(request, limit=reg_limit, window=reg_window) logger.info( - f'{request_id} | Register request from: {request.client.host}:{request.client.port}' + f'Register request from: {request.client.host}:{request.client.port}' ) try: @@ -362,7 +362,7 @@ async def register(request: Request): return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -373,7 +373,7 @@ async def register(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -398,9 +398,9 @@ async def admin_revoke_user_tokens(username: str, request: Request): payload = await auth_required(request) admin_user = payload.get('sub') logger.info( - f'{request_id} | Username: {admin_user} | From: {request.client.host}:{request.client.port}' + f'Username: {admin_user} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(admin_user, 'manage_auth'): return respond_rest( ResponseModel( @@ -421,7 +421,7 @@ async def admin_revoke_user_tokens(username: str, request: Request): ) ) except Exception as e: - logger.error(f'{request_id} | Admin check failed: {str(e)}', exc_info=True) + logger.error(f'Admin check failed: {str(e)}', exc_info=True) revoke_all_for_user(username) return respond_rest( ResponseModel( @@ -431,7 +431,7 @@ async def admin_revoke_user_tokens(username: str, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -442,7 +442,7 @@ async def admin_revoke_user_tokens(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -467,9 +467,9 @@ async def admin_unrevoke_user_tokens(username: str, request: Request): payload = await auth_required(request) admin_user = payload.get('sub') logger.info( - f'{request_id} | Username: {admin_user} | From: {request.client.host}:{request.client.port}' + f'Username: {admin_user} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(admin_user, 'manage_auth'): return respond_rest( ResponseModel( @@ -490,7 +490,7 @@ async def admin_unrevoke_user_tokens(username: str, request: Request): ) ) except Exception as e: - logger.error(f'{request_id} | Admin check failed: {str(e)}', exc_info=True) + logger.error(f'Admin check failed: {str(e)}', exc_info=True) unrevoke_all_for_user(username) return respond_rest( ResponseModel( @@ -500,7 +500,7 @@ async def admin_unrevoke_user_tokens(username: str, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -511,7 +511,7 @@ async def admin_unrevoke_user_tokens(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -536,9 +536,9 @@ async def admin_disable_user(username: str, request: Request): payload = await auth_required(request) admin_user = payload.get('sub') logger.info( - f'{request_id} | Username: {admin_user} | From: {request.client.host}:{request.client.port}' + f'Username: {admin_user} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(admin_user, 'manage_auth'): return respond_rest( ResponseModel( @@ -559,7 +559,7 @@ async def admin_disable_user(username: str, request: Request): ) ) except Exception as e: - logger.error(f'{request_id} | Admin check failed: {str(e)}', exc_info=True) + logger.error(f'Admin check failed: {str(e)}', exc_info=True) await UserService.update_user(username, UpdateUserModel(active=False), request_id) @@ -572,7 +572,7 @@ async def admin_disable_user(username: str, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -583,7 +583,7 @@ async def admin_disable_user(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -608,9 +608,9 @@ async def admin_enable_user(username: str, request: Request): payload = await auth_required(request) admin_user = payload.get('sub') logger.info( - f'{request_id} | Username: {admin_user} | From: {request.client.host}:{request.client.port}' + f'Username: {admin_user} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(admin_user, 'manage_auth'): return respond_rest( ResponseModel( @@ -631,7 +631,7 @@ async def admin_enable_user(username: str, request: Request): ) ) except Exception as e: - logger.error(f'{request_id} | Admin check failed: {str(e)}', exc_info=True) + logger.error(f'Admin check failed: {str(e)}', exc_info=True) await UserService.update_user(username, UpdateUserModel(active=True), request_id) return respond_rest( @@ -642,7 +642,7 @@ async def admin_enable_user(username: str, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -653,7 +653,7 @@ async def admin_enable_user(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -678,9 +678,9 @@ async def admin_user_status(username: str, request: Request): payload = await auth_required(request) admin_user = payload.get('sub') logger.info( - f'{request_id} | Username: {admin_user} | From: {request.client.host}:{request.client.port}' + f'Username: {admin_user} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(admin_user, 'manage_auth'): return respond_rest( ResponseModel( @@ -701,7 +701,7 @@ async def admin_user_status(username: str, request: Request): ) ) except Exception as e: - logger.error(f'{request_id} | Admin check failed: {str(e)}', exc_info=True) + logger.error(f'Admin check failed: {str(e)}', exc_info=True) user = await UserService.get_user_by_username_helper(username) status = {'active': bool(user.get('active', False)), 'revoked': is_user_revoked(username)} return respond_rest( @@ -710,7 +710,7 @@ async def admin_user_status(username: str, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -721,7 +721,7 @@ async def admin_user_status(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -752,9 +752,9 @@ async def extended_authorization(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') user = await UserService.get_user_by_username_helper(username) if not user['active']: return respond_rest( @@ -795,7 +795,7 @@ async def extended_authorization(request: Request): if not _secure and os.getenv('ENV', '').lower() in ('production', 'prod'): logger.warning( - f'{request_id} | SECURITY WARNING: Secure cookies disabled in production environment' + f'SECURITY WARNING: Secure cookies disabled in production environment' ) _domain = os.getenv('COOKIE_DOMAIN', None) @@ -873,7 +873,7 @@ async def extended_authorization(request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -884,7 +884,7 @@ async def extended_authorization(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -915,9 +915,9 @@ async def authorization_status(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest( ResponseModel( status_code=200, @@ -944,7 +944,7 @@ async def authorization_status(request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -955,7 +955,7 @@ async def authorization_status(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -988,9 +988,9 @@ async def authorization_invalidate(response: Response, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') try: import time as _t @@ -1000,7 +1000,7 @@ async def authorization_invalidate(response: Response, request: Request): ttl = max(1, int(exp - _t.time())) add_revoked_jti(username, payload.get('jti'), ttl) except Exception as e: - logger.warning(f'{request_id} | Token revocation failed, using fallback: {str(e)}') + logger.warning(f'Token revocation failed, using fallback: {str(e)}') if username not in jwt_blacklist: jwt_blacklist[username] = TimedHeap() jwt_blacklist[username].push(payload.get('jti')) @@ -1037,7 +1037,7 @@ async def authorization_invalidate(response: Response, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -1048,4 +1048,4 @@ async def authorization_invalidate(response: Response, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/config_routes.py b/backend-services/routes/config_routes.py index a8875bf7..e493744d 100644 --- a/backend-services/routes/config_routes.py +++ b/backend-services/routes/config_routes.py @@ -188,7 +188,7 @@ async def export_all(request: Request): except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_all error: {e}') + logger.error(f'export_all error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -196,7 +196,7 @@ async def export_all(request: Request): 'rest', ) finally: - logger.info(f'{request_id} | export_all took {time.time() * 1000 - start:.2f}ms') + logger.info(f'export_all took {time.time() * 1000 - start:.2f}ms') """ @@ -274,7 +274,7 @@ async def export_apis( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_apis error: {e}') + logger.error(f'export_apis error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -282,7 +282,7 @@ async def export_apis( 'rest', ) finally: - logger.info(f'{request_id} | export_apis took {time.time() * 1000 - start:.2f}ms') + logger.info(f'export_apis took {time.time() * 1000 - start:.2f}ms') """ @@ -345,7 +345,7 @@ async def export_roles(request: Request, role_name: str | None = None): except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_roles error: {e}') + logger.error(f'export_roles error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -416,7 +416,7 @@ async def export_groups(request: Request, group_name: str | None = None): except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_groups error: {e}') + logger.error(f'export_groups error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -488,7 +488,7 @@ async def export_routings(request: Request, client_key: str | None = None): except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_routings error: {e}') + logger.error(f'export_routings error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -539,7 +539,7 @@ async def export_endpoints( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | export_endpoints error: {e}') + logger.error(f'export_endpoints error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -712,7 +712,7 @@ async def import_all(request: Request, body: dict[str, Any]): except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | import_all error: {e}') + logger.error(f'import_all error: {e}') return process_response( ResponseModel( status_code=500, error_code='GTW999', error_message='An unexpected error occurred' @@ -720,7 +720,7 @@ async def import_all(request: Request, body: dict[str, Any]): 'rest', ) finally: - logger.info(f'{request_id} | import_all took {time.time() * 1000 - start:.2f}ms') + logger.info(f'import_all took {time.time() * 1000 - start:.2f}ms') @config_router.post( @@ -762,5 +762,5 @@ async def rollback_config(request: Request): 'rest' ) except Exception as e: - logger.error(f'{request_id} | rollback error: {e}') + logger.error(f'rollback error: {e}') raise HTTPException(status_code=500, detail='Rollback failed') diff --git a/backend-services/routes/credit_routes.py b/backend-services/routes/credit_routes.py index a59c1872..3158d572 100644 --- a/backend-services/routes/credit_routes.py +++ b/backend-services/routes/credit_routes.py @@ -46,9 +46,9 @@ async def list_credit_definitions(request: Request, page: int = 1, page_size: in payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -71,7 +71,7 @@ async def list_credit_definitions(request: Request, page: int = 1, page_size: in except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -83,7 +83,7 @@ async def list_credit_definitions(request: Request, page: int = 1, page_size: in ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -106,9 +106,9 @@ async def get_credit_definition(api_credit_group: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -117,7 +117,7 @@ async def get_credit_definition(api_credit_group: str, request: Request): ) return respond_rest(await CreditService.get_credit_def(api_credit_group, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -129,7 +129,7 @@ async def get_credit_definition(api_credit_group: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -164,9 +164,9 @@ async def create_credit(credit_data: CreditModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -187,7 +187,7 @@ async def create_credit(credit_data: CreditModel, request: Request): ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -199,7 +199,7 @@ async def create_credit(credit_data: CreditModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -234,9 +234,9 @@ async def update_credit(api_credit_group: str, credit_data: CreditModel, request payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -257,7 +257,7 @@ async def update_credit(api_credit_group: str, credit_data: CreditModel, request ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -269,7 +269,7 @@ async def update_credit(api_credit_group: str, credit_data: CreditModel, request ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -304,9 +304,9 @@ async def delete_credit(api_credit_group: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -327,7 +327,7 @@ async def delete_credit(api_credit_group: str, request: Request): ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -339,7 +339,7 @@ async def delete_credit(api_credit_group: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -370,9 +370,9 @@ async def add_user_credits(username: str, credit_data: UserCreditModel, request: payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -393,7 +393,7 @@ async def add_user_credits(username: str, credit_data: UserCreditModel, request: ) return respond_rest(result) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -405,7 +405,7 @@ async def add_user_credits(username: str, credit_data: UserCreditModel, request: ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -428,9 +428,9 @@ async def get_all_users_credits( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_credits'): return respond_rest( ResponseModel( @@ -443,7 +443,7 @@ async def get_all_users_credits( await CreditService.get_all_credits(page, page_size, request_id, search=search) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -455,7 +455,7 @@ async def get_all_users_credits( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -490,7 +490,7 @@ async def get_credits(username: str, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -501,7 +501,7 @@ async def get_credits(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -549,16 +549,16 @@ async def rotate_key(request: Request): ) logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # No special role required, just authentication return respond_rest(await CreditService.rotate_api_key(username, group, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -569,4 +569,4 @@ async def rotate_key(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/dashboard_routes.py b/backend-services/routes/dashboard_routes.py index 29cb5e79..a939a1af 100644 --- a/backend-services/routes/dashboard_routes.py +++ b/backend-services/routes/dashboard_routes.py @@ -14,7 +14,7 @@ from models.response_model import ResponseModel from utils.auth_util import auth_required from utils.database import api_collection, subscriptions_collection, user_collection -from utils.metrics_util import metrics_store +from utils.enhanced_metrics_util import enhanced_metrics_store from utils.response_util import respond_rest dashboard_router = APIRouter() @@ -39,14 +39,18 @@ async def get_dashboard_data(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') - total_users = user_collection.count_documents({'active': True}) total_apis = api_collection.count_documents({}) - snap = metrics_store.snapshot('30d') + snap = enhanced_metrics_store.snapshot('30d') + # Prefer calculated unique users for the period, fallback to top users count + total_users = snap.get('unique_users') + if total_users is None: + # Fallback: count from top_users (which might be truncated) + total_users = len(snap.get('top_users', [])) monthly_usage: dict[str, int] = {} for pt in snap.get('series', []): try: @@ -110,7 +114,7 @@ async def get_dashboard_data(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -121,4 +125,4 @@ async def get_dashboard_data(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/demo_routes.py b/backend-services/routes/demo_routes.py index 3bd00fd7..07176426 100644 --- a/backend-services/routes/demo_routes.py +++ b/backend-services/routes/demo_routes.py @@ -47,9 +47,9 @@ async def demo_seed( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await is_admin_user(username): return respond_rest( @@ -72,7 +72,7 @@ async def demo_seed( except HTTPException as e: raise e except Exception as e: - logger.error(f'{request_id} | Demo seed error: {str(e)}', exc_info=True) + logger.error(f'Demo seed error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, error_code='DEMO999', error_message='Failed to seed demo data' @@ -80,4 +80,4 @@ async def demo_seed( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/endpoint_routes.py b/backend-services/routes/endpoint_routes.py index 1bc1298e..fd0cb8ee 100644 --- a/backend-services/routes/endpoint_routes.py +++ b/backend-services/routes/endpoint_routes.py @@ -57,9 +57,9 @@ async def create_endpoint(endpoint_data: CreateEndpointModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -73,7 +73,7 @@ async def create_endpoint(endpoint_data: CreateEndpointModel, request: Request): except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -85,7 +85,7 @@ async def create_endpoint(endpoint_data: CreateEndpointModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -125,9 +125,9 @@ async def update_endpoint( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -150,7 +150,7 @@ async def update_endpoint( except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -162,7 +162,7 @@ async def update_endpoint( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -197,9 +197,9 @@ async def delete_endpoint( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -217,7 +217,7 @@ async def delete_endpoint( except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -229,7 +229,7 @@ async def delete_endpoint( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -256,9 +256,9 @@ async def get_endpoint( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest( await EndpointService.get_endpoint( endpoint_method, api_name, api_version, '/' + endpoint_uri, request_id @@ -267,7 +267,7 @@ async def get_endpoint( except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -279,7 +279,7 @@ async def get_endpoint( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -304,16 +304,16 @@ async def get_endpoints_by_name_version(api_name: str, api_version: str, request payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest( await EndpointService.get_endpoints_by_name_version(api_name, api_version, request_id) ) except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -325,7 +325,7 @@ async def get_endpoints_by_name_version(api_name: str, api_version: str, request ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -362,9 +362,9 @@ async def create_endpoint_validation( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -380,7 +380,7 @@ async def create_endpoint_validation( except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -392,7 +392,7 @@ async def create_endpoint_validation( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -419,9 +419,9 @@ async def update_endpoint_validation( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -439,7 +439,7 @@ async def update_endpoint_validation( except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -451,7 +451,7 @@ async def update_endpoint_validation( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -476,9 +476,9 @@ async def delete_endpoint_validation(endpoint_id: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ENDPOINTS): return respond_rest( ResponseModel( @@ -494,7 +494,7 @@ async def delete_endpoint_validation(endpoint_id: str, request: Request): except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -506,7 +506,7 @@ async def delete_endpoint_validation(endpoint_id: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -531,14 +531,14 @@ async def get_endpoint_validation(endpoint_id: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await EndpointService.get_endpoint_validation(endpoint_id, request_id)) except HTTPException as he: raise he except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -550,7 +550,7 @@ async def get_endpoint_validation(endpoint_id: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ diff --git a/backend-services/routes/gateway_routes.py b/backend-services/routes/gateway_routes.py index 3f3b1743..a9f392c0 100644 --- a/backend-services/routes/gateway_routes.py +++ b/backend-services/routes/gateway_routes.py @@ -108,7 +108,7 @@ async def status(request: Request): ).dict(), 'rest', ) - logger.error(f'{request_id} | Status check failed: {str(e)}') + logger.error(f'Status check failed: {str(e)}') return process_response( ResponseModel( status_code=500, @@ -120,7 +120,7 @@ async def status(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Status check time {end_time - start_time}ms') + logger.info(f'Status check time {end_time - start_time}ms') @gateway_router.get('/health', description='Public health probe', include_in_schema=False) @@ -238,7 +238,7 @@ async def clear_all_caches(request: Request): return resp finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Clear caches took {end_time - start_time:.2f}ms') + logger.info(f'Clear caches took {end_time - start_time:.2f}ms') # Handle CORS preflight without requiring auth @@ -297,7 +297,7 @@ async def _resolve_api_by(name: str, version: str, endpoint_parts: list[str]): ) try: logger.debug( - f"{request_id} | REST route resolve: path={path} key1={key1} key2={key2} api_key={'set' if api_key else 'none'}" + f"REST route resolve: path={path} key1={key1} key2={key2} api_key={'set' if api_key else 'none'}" ) except Exception: pass @@ -393,17 +393,18 @@ async def _resolve_api_by(name: str, version: str, endpoint_parts: list[str]): else: pass logger.info( - f'{request_id} | Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' + f'Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' ) - logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' - ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + if username: + logger.info( + f'Username: {username} | From: {request.client.host}:{request.client.port}' + ) return process_response( await GatewayService.rest_gateway(username, request, request_id, start_time, path), 'rest', ) except HTTPException as e: + logger.warning(f"Request failed (HTTP {e.status_code}): {e.detail}") return process_response( ResponseModel( status_code=e.status_code, @@ -414,7 +415,7 @@ async def _resolve_api_by(name: str, version: str, endpoint_parts: list[str]): 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -426,7 +427,7 @@ async def _resolve_api_by(name: str, version: str, endpoint_parts: list[str]): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @gateway_router.get( @@ -599,7 +600,7 @@ async def rest_preflight(request: Request, path: str): return StarletteResponse(status_code=204, headers={'request_id': request_id}) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -699,12 +700,12 @@ async def soap_gateway(request: Request, path: str): else: pass logger.info( - f'{request_id} | Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' + f'Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' ) logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return process_response( await GatewayService.soap_gateway(username, request, request_id, start_time, path), 'soap', @@ -720,7 +721,7 @@ async def soap_gateway(request: Request, path: str): 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -732,7 +733,7 @@ async def soap_gateway(request: Request, path: str): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @gateway_router.api_route( @@ -801,7 +802,7 @@ async def grpc_gateway(request: Request, path: str): from services.user_service import UserService as _US u = await _US.get_user_by_username_helper(username) if (u.get('role') or '') not in set(allowed_roles): - return process_response( + return process_response( ResponseModel( status_code=403, response_headers={'request_id': request_id}, @@ -915,7 +916,7 @@ async def soap_preflight(request: Request, path: str): return StarletteResponse(status_code=204, headers={'request_id': request_id}) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -1011,12 +1012,12 @@ async def graphql_gateway(request: Request, path: str): else: pass logger.info( - f'{request_id} | Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' + f'Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' ) logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if api and api.get('validation_enabled'): body = await request.json() query = body.get('query') @@ -1048,7 +1049,7 @@ async def graphql_gateway(request: Request, path: str): 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -1060,7 +1061,7 @@ async def graphql_gateway(request: Request, path: str): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -1132,7 +1133,7 @@ async def graphql_preflight(request: Request, path: str): return StarletteResponse(status_code=204, headers={'request_id': request_id}) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @gateway_router.api_route( @@ -1205,7 +1206,7 @@ async def grpc_preflight(request: Request, path: str): return StarletteResponse(status_code=204, headers={'request_id': request_id}) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -1227,3 +1228,115 @@ async def grpc_preflight(request: Request, path: str): """ +@gateway_router.api_route( + '/grpc/{path:path}', + methods=['POST', 'GET'], + description='gRPC gateway endpoint', + response_model=ResponseModel, +) +async def grpc_gateway(request: Request, path: str): + request_id = ( + getattr(request.state, 'request_id', None) + or request.headers.get('X-Request-ID') + or str(uuid.uuid4()) + ) + start_time = time.time() * 1000 + try: + # Determine API context for Auth/Subscription + parts = [p for p in (path or '').split('/') if p] + api_name = parts[-1] if parts else None + + # Default behavior if api_name not found (GatewayService will handle 404/errors later) + # But we need it for auth. + api = None + username = None + api_public = False + api_auth_required = True + + if api_name: + api_version = request.headers.get('X-API-Version', 'v1') + api_path = f'{api_name}/{api_version}' + api = doorman_cache.get_cache('api_cache', api_path) + if not api: + api = await api_util.get_api(None, api_path) + + if api: + api_public = bool(api.get('api_public')) + api_auth_required = ( + bool(api.get('api_auth_required')) + if api.get('api_auth_required') is not None + else True + ) + try: + enforce_api_ip_policy(request, api) + except HTTPException as e: + return process_response( + ResponseModel( + status_code=e.status_code, + error_code=e.detail, + error_message='IP restricted', + ).dict(), + 'grpc', + ) + + if api and not api_public: + if api_auth_required: + try: + await subscription_required(request) + await group_required(request) + await limit_and_throttle(request) + payload = await auth_required(request) + username = payload.get('sub') + + # Enforce API allowed roles + allowed_roles = api.get('api_allowed_roles') or [] + if allowed_roles: + from services.user_service import UserService as _US + u = await _US.get_user_by_username_helper(username) + if (u.get('role') or '') not in set(allowed_roles): + return process_response( + ResponseModel( + status_code=403, + response_headers={'request_id': request_id}, + error_code='GTW014', + error_message='Forbidden: role not allowed for this API', + ).dict(), + 'grpc', + ) + + await enforce_pre_request_limit(request, username) + except HTTPException as e: + # Map HTTP exceptions (like 403 from sub check, 429 from limits) to gRPC response? + # process_response with 'grpc' will handle it. + return process_response( + ResponseModel( + status_code=e.status_code, + error_code=e.detail, # detail usually string or dict + error_message=str(e.detail), + ).dict(), + 'grpc', + ) + + logger.info( + f'{request_id} | Time: {datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")[:-3]}ms' + ) + logger.info( + f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + ) + logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + + return process_response( + await GatewayService.grpc_gateway(username, request, request_id, start_time, path), + 'grpc', + ) + except Exception as e: + logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + return process_response( + ResponseModel( + status_code=500, + response_headers={'request_id': request_id}, + error_code='GTW999', + error_message='An unexpected error occurred', + ).dict(), + 'grpc', + ) diff --git a/backend-services/routes/graphql_routes.py b/backend-services/routes/graphql_routes.py index 05252fa5..3ac946bb 100644 --- a/backend-services/routes/graphql_routes.py +++ b/backend-services/routes/graphql_routes.py @@ -104,7 +104,7 @@ async def get_graphql_schema(api_name: str, api_version: str, request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error getting GraphQL schema: {e}', exc_info=True) + logger.error(f'Error getting GraphQL schema: {e}', exc_info=True) return respond_rest('GQL999', str(e), None, None, 500, start_time) @@ -168,7 +168,7 @@ async def refresh_graphql_schema(api_name: str, api_version: str, request: Reque ) except Exception as e: - logger.error(f'{request_id} | Error refreshing GraphQL schema: {e}', exc_info=True) + logger.error(f'Error refreshing GraphQL schema: {e}', exc_info=True) return respond_rest('GQL999', str(e), None, None, 500, start_time) @@ -220,6 +220,5 @@ async def get_graphql_types(api_name: str, api_version: str, request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error getting GraphQL types: {e}', exc_info=True) + logger.error(f'Error getting GraphQL types: {e}', exc_info=True) return respond_rest('GQL999', str(e), None, None, 500, start_time) - diff --git a/backend-services/routes/group_routes.py b/backend-services/routes/group_routes.py index 8cd54e89..1880d605 100644 --- a/backend-services/routes/group_routes.py +++ b/backend-services/routes/group_routes.py @@ -52,9 +52,9 @@ async def create_group(api_data: CreateGroupModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_GROUPS): return respond_rest( ResponseModel( @@ -68,7 +68,7 @@ async def create_group(api_data: CreateGroupModel, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -79,7 +79,7 @@ async def create_group(api_data: CreateGroupModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -110,9 +110,9 @@ async def update_group(group_name: str, api_data: UpdateGroupModel, request: Req payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_GROUPS): return respond_rest( ResponseModel( @@ -126,7 +126,7 @@ async def update_group(group_name: str, api_data: UpdateGroupModel, request: Req except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -137,7 +137,7 @@ async def update_group(group_name: str, api_data: UpdateGroupModel, request: Req ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -168,9 +168,9 @@ async def delete_group(group_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_GROUPS): return respond_rest( ResponseModel( @@ -184,7 +184,7 @@ async def delete_group(group_name: str, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -195,7 +195,7 @@ async def delete_group(group_name: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -218,14 +218,14 @@ async def get_groups( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await GroupService.get_groups(page, page_size, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -237,7 +237,7 @@ async def get_groups( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -258,14 +258,14 @@ async def get_group(group_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await GroupService.get_group(group_name, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -277,4 +277,4 @@ async def get_group(group_name: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/grpc_routes.py b/backend-services/routes/grpc_routes.py index b7ae10db..97e10582 100644 --- a/backend-services/routes/grpc_routes.py +++ b/backend-services/routes/grpc_routes.py @@ -83,7 +83,7 @@ async def list_grpc_services(api_name: str, api_version: str, request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error listing gRPC services: {e}', exc_info=True) + logger.error(f'Error listing gRPC services: {e}', exc_info=True) return respond_rest('GRPC999', str(e), None, None, 500, start_time) @@ -104,7 +104,7 @@ async def grpc_web_proxy(api_name: str, service: str, method: str, request: Requ and responses back to gRPC-Web frames. """ request_id = str(uuid.uuid4()) - logger.info(f'{request_id} | gRPC-Web request: {api_name} -> {service}/{method}') + logger.info(f'gRPC-Web request: {api_name} -> {service}/{method}') # Check headers content_type = request.headers.get('content-type', '') @@ -158,7 +158,7 @@ async def grpc_web_proxy(api_name: str, service: str, method: str, request: Requ # Since we don't have a full gRPC client setup here that can handle raw bytes, # we'll stub the response to prove the route infrastructure works. - logger.info(f'{request_id} | gRPC-Web proxy to {servers[0]}') + logger.info(f'gRPC-Web proxy to {servers[0]}') # Simulate successful empty response resp_body = create_grpc_web_response(b'', {'grpc-status': '0', 'grpc-message': 'OK'}) @@ -172,5 +172,5 @@ async def grpc_web_proxy(api_name: str, service: str, method: str, request: Requ ) except Exception as e: - logger.error(f'{request_id} | gRPC-Web error: {e}', exc_info=True) + logger.error(f'gRPC-Web error: {e}', exc_info=True) return Response(status_code=500) diff --git a/backend-services/routes/logging_routes.py b/backend-services/routes/logging_routes.py index e88c1ec2..88fca7ac 100644 --- a/backend-services/routes/logging_routes.py +++ b/backend-services/routes/logging_routes.py @@ -85,6 +85,7 @@ async def get_logs( level: str | None = Query(None, description='Filter by log level'), limit: int = Query(100, description='Number of logs to return', ge=1, le=1000), offset: int = Query(0, description='Number of logs to skip', ge=0), + type: str | None = Query(None, description='Filter by request type (platform, auth, gateway)'), ): request_id_param = str(uuid.uuid4()) start_time_param = time.time() * 1000 @@ -125,6 +126,8 @@ async def get_logs( limit=limit, offset=offset, request_id_param=request_id_param, + type=type, + exclude_type=request.query_params.get('exclude_type'), ) return respond_rest( @@ -170,9 +173,9 @@ async def get_log_files(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.VIEW_LOGS): return respond_rest( @@ -198,7 +201,7 @@ async def get_log_files(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -209,7 +212,7 @@ async def get_log_files(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -263,9 +266,9 @@ async def get_log_statistics(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.VIEW_LOGS): return respond_rest( @@ -289,7 +292,7 @@ async def get_log_statistics(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -301,7 +304,7 @@ async def get_log_statistics(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -342,6 +345,7 @@ async def export_logs( api: str | None = Query(None, description='Filter by API'), endpoint: str | None = Query(None, description='Filter by endpoint'), level: str | None = Query(None, description='Filter by log level'), + type: str | None = Query(None, description='Filter by request type (platform, auth, gateway)'), ): request_id = str(uuid.uuid4()) start_time = time.time() * 1000 @@ -349,9 +353,9 @@ async def export_logs( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.EXPORT_LOGS): return process_response( @@ -375,6 +379,8 @@ async def export_logs( filters['endpoint'] = endpoint if level: filters['level'] = level + if type: + filters['type'] = type export_result = await logging_service.export_logs( format=format, @@ -393,7 +399,7 @@ async def export_logs( ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -405,7 +411,7 @@ async def export_logs( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -435,6 +441,7 @@ async def download_logs( api: str | None = Query(None, description='Filter by API'), endpoint: str | None = Query(None, description='Filter by endpoint'), level: str | None = Query(None, description='Filter by log level'), + type: str | None = Query(None, description='Filter by request type (platform, auth, gateway)'), ): request_id = str(uuid.uuid4()) start_time = time.time() * 1000 @@ -442,9 +449,9 @@ async def download_logs( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.EXPORT_LOGS): return process_response( @@ -468,6 +475,8 @@ async def download_logs( filters['endpoint'] = endpoint if level: filters['level'] = level + if type: + filters['type'] = type export_result = await logging_service.export_logs( format=format, @@ -492,7 +501,7 @@ async def download_logs( ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -504,4 +513,4 @@ async def download_logs( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/memory_routes.py b/backend-services/routes/memory_routes.py index 442b675f..2f355770 100644 --- a/backend-services/routes/memory_routes.py +++ b/backend-services/routes/memory_routes.py @@ -47,9 +47,9 @@ async def memory_dump(request: Request, body: DumpRequest | None = None): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( @@ -100,7 +100,7 @@ async def memory_dump(request: Request, body: DumpRequest | None = None): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -112,7 +112,7 @@ async def memory_dump(request: Request, body: DumpRequest | None = None): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') class RestoreRequest(BaseModel): @@ -141,9 +141,9 @@ async def memory_restore(request: Request, body: RestoreRequest | None = None): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( @@ -204,7 +204,7 @@ async def memory_restore(request: Request, body: RestoreRequest | None = None): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -216,4 +216,4 @@ async def memory_restore(request: Request, body: RestoreRequest | None = None): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/monitor_routes.py b/backend-services/routes/monitor_routes.py index 1f45a5d6..5a851c82 100644 --- a/backend-services/routes/monitor_routes.py +++ b/backend-services/routes/monitor_routes.py @@ -60,9 +60,9 @@ async def get_metrics( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_gateway'): return process_response( ResponseModel( @@ -121,7 +121,7 @@ def _name(x): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -133,7 +133,7 @@ def _name(x): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -448,7 +448,7 @@ def _api_from_endpoint(ep: str) -> str: except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error in report: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error in report: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -460,4 +460,4 @@ def _api_from_endpoint(ep: str) -> str: ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/openapi_routes.py b/backend-services/routes/openapi_routes.py index e12bfc57..6824e570 100644 --- a/backend-services/routes/openapi_routes.py +++ b/backend-services/routes/openapi_routes.py @@ -191,7 +191,7 @@ async def get_openapi_spec(api_name: str, api_version: str, request: Request): return respond_rest(None, None, spec, request_id, 200, start_time) except Exception as e: - logger.error(f'{request_id} | Error getting OpenAPI spec: {e}', exc_info=True) + logger.error(f'Error getting OpenAPI spec: {e}', exc_info=True) return respond_rest('OPENAPI999', str(e), None, None, 500, start_time) @@ -268,7 +268,7 @@ async def refresh_openapi_spec(api_name: str, api_version: str, request: Request ) except Exception as e: - logger.error(f'{request_id} | Error refreshing OpenAPI spec: {e}', exc_info=True) + logger.error(f'Error refreshing OpenAPI spec: {e}', exc_info=True) return respond_rest('OPENAPI999', str(e), None, None, 500, start_time) @@ -372,7 +372,7 @@ async def import_openapi_endpoints(api_name: str, api_version: str, request: Req ) except Exception as e: - logger.error(f'{request_id} | Error importing OpenAPI endpoints: {e}', exc_info=True) + logger.error(f'Error importing OpenAPI endpoints: {e}', exc_info=True) return respond_rest('OPENAPI999', str(e), None, None, 500, start_time) @@ -423,5 +423,5 @@ async def parse_openapi_spec(request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error parsing OpenAPI spec: {e}', exc_info=True) + logger.error(f'Error parsing OpenAPI spec: {e}', exc_info=True) return respond_rest('OPENAPI999', str(e), None, None, 500, start_time) diff --git a/backend-services/routes/proto_routes.py b/backend-services/routes/proto_routes.py index ea115bc6..d57ce7b1 100644 --- a/backend-services/routes/proto_routes.py +++ b/backend-services/routes/proto_routes.py @@ -273,8 +273,8 @@ async def upload_proto_file( pass payload = await auth_required(request) username = payload.get('sub') - logger.info(f'{request_id} | Username: {username}') - logger.info(f'{request_id} | Endpoint: POST /proto/{api_name}/{api_version}') + logger.info(f'Username: {username}') + logger.info(f'Endpoint: POST /proto/{api_name}/{api_version}') if not await platform_role_required_bool(username, Roles.MANAGE_APIS): return process_response( ResponseModel( @@ -393,7 +393,7 @@ async def upload_proto_file( ], check=True, ) - logger.info(f'{request_id} | Proto compiled: src={compile_input} out={generated_dir}') + logger.info(f'Proto compiled: src={compile_input} out={generated_dir}') init_path = (generated_dir / '__init__.py').resolve() if not validate_path(generated_dir, init_path): raise ValueError('Invalid init path') @@ -435,7 +435,7 @@ async def upload_proto_file( 'rest', ) except subprocess.CalledProcessError as e: - logger.error(f'{request_id} | Failed to generate gRPC code: {str(e)}') + logger.error(f'Failed to generate gRPC code: {str(e)}') return process_response( ResponseModel( status_code=500, @@ -446,7 +446,7 @@ async def upload_proto_file( 'rest', ) except HTTPException as e: - logger.error(f'{request_id} | Path validation error: {str(e)}') + logger.error(f'Path validation error: {str(e)}') return process_response( ResponseModel( status_code=e.status_code, @@ -458,7 +458,7 @@ async def upload_proto_file( ) except Exception as e: logger.error( - f'{request_id} | Error uploading proto file: {type(e).__name__}: {str(e)}', + f'Error uploading proto file: {type(e).__name__}: {str(e)}', exc_info=True, ) return process_response( @@ -471,7 +471,7 @@ async def upload_proto_file( 'rest', ) finally: - logger.info(f'{request_id} | Total time: {time.time() * 1000 - start_time}ms') + logger.info(f'Total time: {time.time() * 1000 - start_time}ms') """ @@ -502,8 +502,8 @@ async def get_proto_file(api_name: str, api_version: str, request: Request): start_time = time.time() * 1000 payload = await auth_required(request) username = payload.get('sub') - logger.info(f'{request_id} | Username: {username} | From: {request.client.host}') - logger.info(f'{request_id} | Endpoint: {request.method} {request.url.path}') + logger.info(f'Username: {username} | From: {request.client.host}') + logger.info(f'Endpoint: {request.method} {request.url.path}') try: if not await platform_role_required_bool(username, Roles.MANAGE_APIS): return process_response( @@ -539,7 +539,7 @@ async def get_proto_file(api_name: str, api_version: str, request: Request): 'rest', ) except HTTPException as e: - logger.error(f'{request_id} | Path validation error: {str(e)}') + logger.error(f'Path validation error: {str(e)}') return process_response( ResponseModel( status_code=e.status_code, @@ -550,7 +550,7 @@ async def get_proto_file(api_name: str, api_version: str, request: Request): 'rest', ) except Exception as e: - logger.error(f'{request_id} | Failed to get proto file: {str(e)}') + logger.error(f'Failed to get proto file: {str(e)}') return process_response( ResponseModel( status_code=500, @@ -561,7 +561,7 @@ async def get_proto_file(api_name: str, api_version: str, request: Request): 'rest', ) finally: - logger.info(f'{request_id} | Total time: {time.time() * 1000 - start_time}ms') + logger.info(f'Total time: {time.time() * 1000 - start_time}ms') """ @@ -596,9 +596,9 @@ async def update_proto_file( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_APIS): return process_response( ResponseModel( @@ -670,7 +670,7 @@ async def update_proto_file( check=True, ) except subprocess.CalledProcessError as e: - logger.error(f'{request_id} | Failed to generate gRPC code: {str(e)}') + logger.error(f'Failed to generate gRPC code: {str(e)}') return process_response( ResponseModel( status_code=500, @@ -689,7 +689,7 @@ async def update_proto_file( 'rest', ) except HTTPException as e: - logger.error(f'{request_id} | Path validation error: {str(e)}') + logger.error(f'Path validation error: {str(e)}') return process_response( ResponseModel( status_code=e.status_code, @@ -700,7 +700,7 @@ async def update_proto_file( 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -712,7 +712,7 @@ async def update_proto_file( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -745,9 +745,9 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_APIS): return process_response( ResponseModel( @@ -766,7 +766,7 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): if not validate_path(PROJECT_ROOT, proto_path): raise ValueError('Unsafe proto file path detected') proto_path.unlink() - logger.info(f'{request_id} | Deleted proto file: {proto_path}') + logger.info(f'Deleted proto file: {proto_path}') generated_files = [ f'{key}_pb2.py', f'{key}_pb2.pyc', @@ -777,12 +777,12 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): file_path = (generated_dir / file).resolve() if not validate_path(generated_dir, file_path): logger.warning( - f'{request_id} | Unsafe file path detected: {file_path}. Skipping deletion.' + f'Unsafe file path detected: {file_path}. Skipping deletion.' ) continue if file_path.exists(): file_path.unlink() - logger.info(f'{request_id} | Deleted generated file: {file_path}') + logger.info(f'Deleted generated file: {file_path}') return process_response( ResponseModel( status_code=200, @@ -792,7 +792,7 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): 'rest', ) except ValueError as e: - logger.error(f'{request_id} | Path validation error: {str(e)}') + logger.error(f'Path validation error: {str(e)}') return process_response( ResponseModel( status_code=400, @@ -803,7 +803,7 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -815,4 +815,4 @@ async def delete_proto_file(api_name: str, api_version: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/role_routes.py b/backend-services/routes/role_routes.py index 60106cdf..6c1e189c 100644 --- a/backend-services/routes/role_routes.py +++ b/backend-services/routes/role_routes.py @@ -52,11 +52,11 @@ async def create_role(api_data: CreateRoleModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ROLES): - logger.error(f'{request_id} | User does not have permission to create roles') + logger.error(f'User does not have permission to create roles') return respond_rest( ResponseModel( status_code=403, @@ -85,7 +85,7 @@ async def create_role(api_data: CreateRoleModel, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -96,7 +96,7 @@ async def create_role(api_data: CreateRoleModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -125,14 +125,14 @@ async def update_role(role_name: str, api_data: UpdateRoleModel, request: Reques start_time = time.time() * 1000 try: # DEBUG: Log the incoming data - logger.info(f'{request_id} | DEBUG: Received model data: {api_data.dict()}') + logger.info(f'DEBUG: Received model data: {api_data.dict()}') payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ROLES): return respond_rest( ResponseModel( @@ -170,7 +170,7 @@ async def update_role(role_name: str, api_data: UpdateRoleModel, request: Reques except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -181,7 +181,7 @@ async def update_role(role_name: str, api_data: UpdateRoleModel, request: Reques ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -212,9 +212,9 @@ async def delete_role(role_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_ROLES): return respond_rest( ResponseModel( @@ -238,7 +238,7 @@ async def delete_role(role_name: str, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -249,7 +249,7 @@ async def delete_role(role_name: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -272,9 +272,9 @@ async def get_roles( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') data = await RoleService.get_roles(page, page_size, request_id) try: if data.get('status_code') == 200 and isinstance(data.get('response'), dict): @@ -297,7 +297,7 @@ async def get_roles( except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -308,7 +308,7 @@ async def get_roles( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -329,9 +329,9 @@ async def get_role(role_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if await is_admin_role(role_name) and not await is_admin_user(username): return respond_rest( ResponseModel( @@ -355,7 +355,7 @@ async def get_role(role_name: str, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -366,7 +366,7 @@ async def get_role(role_name: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @role_router.get( diff --git a/backend-services/routes/routing_routes.py b/backend-services/routes/routing_routes.py index b9330c91..09f3b5f7 100644 --- a/backend-services/routes/routing_routes.py +++ b/backend-services/routes/routing_routes.py @@ -53,9 +53,9 @@ async def create_routing(api_data: CreateRoutingModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_routings'): return respond_rest( ResponseModel( @@ -69,7 +69,7 @@ async def create_routing(api_data: CreateRoutingModel, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -81,7 +81,7 @@ async def create_routing(api_data: CreateRoutingModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -114,9 +114,9 @@ async def update_routing(client_key: str, api_data: UpdateRoutingModel, request: payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_routings'): return respond_rest( ResponseModel( @@ -130,7 +130,7 @@ async def update_routing(client_key: str, api_data: UpdateRoutingModel, request: except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -142,7 +142,7 @@ async def update_routing(client_key: str, api_data: UpdateRoutingModel, request: ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -175,9 +175,9 @@ async def delete_routing(client_key: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_routings'): return respond_rest( ResponseModel( @@ -191,7 +191,7 @@ async def delete_routing(client_key: str, request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -203,7 +203,7 @@ async def delete_routing(client_key: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -226,9 +226,9 @@ async def get_routings(request: Request, page: int = 1, page_size: int = 10): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_routings'): return respond_rest( ResponseModel( @@ -240,7 +240,7 @@ async def get_routings(request: Request, page: int = 1, page_size: int = 10): ) return respond_rest(await RoutingService.get_routings(page, page_size, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -252,7 +252,7 @@ async def get_routings(request: Request, page: int = 1, page_size: int = 10): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -273,9 +273,9 @@ async def get_routing(client_key: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_routings'): return respond_rest( ResponseModel( @@ -287,7 +287,7 @@ async def get_routing(client_key: str, request: Request): ) return respond_rest(await RoutingService.get_routing(client_key, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -299,4 +299,4 @@ async def get_routing(client_key: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/security_routes.py b/backend-services/routes/security_routes.py index 9de581a8..c03bcbf6 100644 --- a/backend-services/routes/security_routes.py +++ b/backend-services/routes/security_routes.py @@ -46,9 +46,9 @@ async def get_security_settings(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( ResponseModel( @@ -109,7 +109,7 @@ async def get_security_settings(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -121,7 +121,7 @@ async def get_security_settings(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -148,9 +148,9 @@ async def update_security_settings(request: Request, body: SecuritySettingsModel payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( ResponseModel( @@ -199,7 +199,7 @@ async def update_security_settings(request: Request, body: SecuritySettingsModel except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -211,7 +211,7 @@ async def update_security_settings(request: Request, body: SecuritySettingsModel ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -236,9 +236,9 @@ async def restart_gateway(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( ResponseModel( @@ -303,7 +303,7 @@ async def restart_gateway(request: Request): preexec_fn=os.setsid, ) except Exception as e: - logger.error(f'{request_id} | Failed to spawn restarter: {e}') + logger.error(f'Failed to spawn restarter: {e}') return process_response( ResponseModel( status_code=500, @@ -333,7 +333,7 @@ async def restart_gateway(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -345,4 +345,4 @@ async def restart_gateway(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/subscription_routes.py b/backend-services/routes/subscription_routes.py index 7f94d69b..1f16ae43 100644 --- a/backend-services/routes/subscription_routes.py +++ b/backend-services/routes/subscription_routes.py @@ -54,9 +54,9 @@ async def subscribe_api(api_data: SubscribeModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # If targeting a different user, require manage_subscriptions permission if api_data.username and api_data.username != username: @@ -84,7 +84,7 @@ async def subscribe_api(api_data: SubscribeModel, request: Request): target_user = api_data.username or username logger.info( - f'{request_id} | Actor: {username} | Action: subscribe | Target: {target_user} | API: {api_data.api_name}/{api_data.api_version}' + f'Actor: {username} | Action: subscribe | Target: {target_user} | API: {api_data.api_name}/{api_data.api_version}' ) result = await SubscriptionService.subscribe(api_data, request_id) actor_user = username @@ -109,7 +109,7 @@ async def subscribe_api(api_data: SubscribeModel, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -120,7 +120,7 @@ async def subscribe_api(api_data: SubscribeModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -153,9 +153,9 @@ async def unsubscribe_api(api_data: SubscribeModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # If targeting a different user, require manage_subscriptions permission if api_data.username and api_data.username != username: @@ -183,7 +183,7 @@ async def unsubscribe_api(api_data: SubscribeModel, request: Request): target_user = api_data.username or username logger.info( - f'{request_id} | Actor: {username} | Action: unsubscribe | Target: {target_user} | API: {api_data.api_name}/{api_data.api_version}' + f'Actor: {username} | Action: unsubscribe | Target: {target_user} | API: {api_data.api_name}/{api_data.api_version}' ) result = await SubscriptionService.unsubscribe(api_data, request_id) actor_user = username @@ -208,7 +208,7 @@ async def unsubscribe_api(api_data: SubscribeModel, request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -219,7 +219,7 @@ async def unsubscribe_api(api_data: SubscribeModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -250,12 +250,12 @@ async def subscriptions_for_current_user(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await SubscriptionService.get_user_subscriptions(username, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -266,7 +266,7 @@ async def subscriptions_for_current_user(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -297,9 +297,9 @@ async def subscriptions_for_user_by_id(user_id: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # Restrict to self or users with manage_subscriptions permission if user_id != username: @@ -315,7 +315,7 @@ async def subscriptions_for_user_by_id(user_id: str, request: Request): return respond_rest(await SubscriptionService.get_user_subscriptions(user_id, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -326,7 +326,7 @@ async def subscriptions_for_user_by_id(user_id: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -351,9 +351,9 @@ async def available_apis(username: str, request: Request): payload = await auth_required(request) actor = payload.get('sub') logger.info( - f'{request_id} | Username: {actor} | From: {request.client.host}:{request.client.port}' + f'Username: {actor} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') accesses = (payload or {}).get('accesses') or {} can_manage = bool(accesses.get('manage_subscriptions')) @@ -404,7 +404,7 @@ async def available_apis(username: str, request: Request): ) return respond_rest(ResponseModel(status_code=200, response={'apis': allowed})) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -415,4 +415,4 @@ async def available_apis(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/tier_routes.py b/backend-services/routes/tier_routes.py index b02905cb..0e1e9d16 100644 --- a/backend-services/routes/tier_routes.py +++ b/backend-services/routes/tier_routes.py @@ -239,9 +239,9 @@ async def list_tiers( username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_tiers'): return respond_rest( @@ -286,7 +286,7 @@ async def list_tiers( ) except Exception as e: - logger.error(f'{request_id} | Error listing tiers: {e}', exc_info=True) + logger.error(f'Error listing tiers: {e}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -298,7 +298,7 @@ async def list_tiers( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') @tier_router.get('/{tier_id}', response_model=TierResponse) @@ -715,9 +715,9 @@ async def get_all_tier_statistics( username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_tiers'): return respond_rest( @@ -738,7 +738,7 @@ async def get_all_tier_statistics( ) except Exception as e: - logger.error(f'{request_id} | Error getting all tier statistics: {e}', exc_info=True) + logger.error(f'Error getting all tier statistics: {e}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -750,7 +750,7 @@ async def get_all_tier_statistics( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') @tier_router.get('/{tier_id}/statistics', response_model=ResponseModel) @@ -770,9 +770,9 @@ async def get_tier_statistics( username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_tiers'): return respond_rest( @@ -793,7 +793,7 @@ async def get_tier_statistics( ) except Exception as e: - logger.error(f'{request_id} | Error getting tier statistics: {e}', exc_info=True) + logger.error(f'Error getting tier statistics: {e}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -805,4 +805,4 @@ async def get_tier_statistics( finally: end_time = time.time() - logger.info(f'{request_id} | Total time: {(end_time - start_time) * 1000:.2f}ms') + logger.info(f'Total time: {(end_time - start_time) * 1000:.2f}ms') diff --git a/backend-services/routes/tools_routes.py b/backend-services/routes/tools_routes.py index 566c0701..a1e6edb3 100644 --- a/backend-services/routes/tools_routes.py +++ b/backend-services/routes/tools_routes.py @@ -98,9 +98,9 @@ async def cors_check(request: Request, body: CorsCheckRequest): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( ResponseModel( @@ -213,7 +213,7 @@ async def cors_check(request: Request, body: CorsCheckRequest): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -225,7 +225,7 @@ async def cors_check(request: Request, body: CorsCheckRequest): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -250,9 +250,9 @@ async def grpc_env_check(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, 'manage_security'): return process_response( ResponseModel( @@ -306,7 +306,7 @@ async def grpc_env_check(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -318,7 +318,7 @@ async def grpc_env_check(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') class ChaosToggleRequest(BaseModel): @@ -376,7 +376,7 @@ async def chaos_toggle(request: Request, body: ChaosToggleRequest): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -388,7 +388,7 @@ async def chaos_toggle(request: Request, body: ChaosToggleRequest): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @tools_router.get( @@ -421,7 +421,7 @@ async def chaos_stats(request: Request): except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -433,4 +433,4 @@ async def chaos_stats(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') diff --git a/backend-services/routes/user_routes.py b/backend-services/routes/user_routes.py index 60b38582..1c68dc93 100644 --- a/backend-services/routes/user_routes.py +++ b/backend-services/routes/user_routes.py @@ -69,9 +69,9 @@ async def create_user(user_data: CreateUserModel, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') if not await platform_role_required_bool(username, Roles.MANAGE_USERS): return respond_rest( ResponseModel( @@ -91,7 +91,7 @@ async def create_user(user_data: CreateUserModel, request: Request): ) return respond_rest(await UserService.create_user(user_data, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -103,7 +103,7 @@ async def create_user(user_data: CreateUserModel, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -134,9 +134,9 @@ async def update_user(username: str, api_data: UpdateUserModel, request: Request payload = await auth_required(request) auth_username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # Block modifications to bootstrap admin user, except for limited operational fields if username == 'admin': allowed_keys = { @@ -176,6 +176,27 @@ async def update_user(username: str, api_data: UpdateUserModel, request: Request error_message='Can only update your own information', ) ) + # Field-level authorization: when users update their own profile (self-service) + # without manage_users permission, they cannot modify restricted fields + is_self_update = auth_username == username + has_manage_users = await platform_role_required_bool(auth_username, Roles.MANAGE_USERS) + if is_self_update and not has_manage_users: + restricted_fields = {'role', 'groups', 'active', 'username'} + try: + incoming_fields = { + k for k, v in (api_data.dict(exclude_unset=True) or {}).items() if v is not None + } + except Exception: + incoming_fields = set() + attempted_restricted = incoming_fields & restricted_fields + if attempted_restricted: + return respond_rest( + ResponseModel( + status_code=403, + error_code='USR023', + error_message=f'Cannot modify restricted fields without manage_users permission: {", ".join(sorted(attempted_restricted))}', + ) + ) if await _safe_is_admin_user(username) and not await _safe_is_admin_user(auth_username): return respond_rest( ResponseModel( @@ -198,7 +219,7 @@ async def update_user(username: str, api_data: UpdateUserModel, request: Request ) return respond_rest(await UserService.update_user(username, api_data, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -210,7 +231,7 @@ async def update_user(username: str, api_data: UpdateUserModel, request: Request ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -241,9 +262,9 @@ async def delete_user(username: str, request: Request): payload = await auth_required(request) auth_username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # Block any deletion of bootstrap admin user if username == 'admin': return respond_rest( @@ -273,7 +294,7 @@ async def delete_user(username: str, request: Request): ) return respond_rest(await UserService.delete_user(username, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -285,7 +306,7 @@ async def delete_user(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -318,9 +339,9 @@ async def update_user_password(username: str, api_data: UpdatePasswordModel, req payload = await auth_required(request) auth_username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # Block any password changes to bootstrap admin user if username == 'admin': return respond_rest( @@ -344,7 +365,7 @@ async def update_user_password(username: str, api_data: UpdatePasswordModel, req ) return respond_rest(await UserService.update_password(username, api_data, request_id)) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -356,7 +377,7 @@ async def update_user_password(username: str, api_data: UpdatePasswordModel, req ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -377,9 +398,9 @@ async def get_user_by_username(request: Request): payload = await auth_required(request) auth_username = payload.get('sub') logger.info( - f'{request_id} | Username: {auth_username} | From: {request.client.host}:{request.client.port}' + f'Username: {auth_username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await UserService.get_user_by_username(auth_username, request_id)) except HTTPException as e: return respond_rest( @@ -391,7 +412,7 @@ async def get_user_by_username(request: Request): ) ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return respond_rest( ResponseModel( status_code=500, @@ -402,7 +423,7 @@ async def get_user_by_username(request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -425,9 +446,9 @@ async def get_all_users( payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') data = await UserService.get_all_users(page, page_size, request_id) if data.get('status_code') == 200 and isinstance(data.get('response'), dict): users = data['response'].get('users') or [] @@ -456,7 +477,7 @@ async def get_all_users( 'rest', ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -468,7 +489,7 @@ async def get_all_users( ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -491,9 +512,9 @@ async def get_user_by_username(username: str, request: Request): payload = await auth_required(request) auth_username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') # Block access to bootstrap admin user for non-admins, # except when STRICT_RESPONSE_ENVELOPE=true (envelope-shape tests) if ( @@ -533,7 +554,7 @@ async def get_user_by_username(username: str, request: Request): await UserService.get_user_by_username(username, request_id), 'rest' ) except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -545,7 +566,7 @@ async def get_user_by_username(username: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') """ @@ -568,9 +589,9 @@ async def get_user_by_email(email: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') data = await UserService.get_user_by_email(username, email, request_id) if data.get('status_code') == 200 and isinstance(data.get('response'), dict): u = data.get('response') @@ -596,7 +617,7 @@ async def get_user_by_email(email: str, request: Request): ) return process_response(data, 'rest') except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -608,7 +629,7 @@ async def get_user_by_email(email: str, request: Request): ) finally: end_time = time.time() * 1000 - logger.info(f'{request_id} | Total time: {str(end_time - start_time)}ms') + logger.info(f'Total time: {str(end_time - start_time)}ms') @user_router.get( diff --git a/backend-services/routes/vault_routes.py b/backend-services/routes/vault_routes.py index abc63e24..475396dc 100644 --- a/backend-services/routes/vault_routes.py +++ b/backend-services/routes/vault_routes.py @@ -52,15 +52,15 @@ async def create_vault_entry(entry_data: CreateVaultEntryModel, request: Request payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await VaultService.create_vault_entry(username, entry_data, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -71,7 +71,7 @@ async def create_vault_entry(entry_data: CreateVaultEntryModel, request: Request ) finally: elapsed = time.time() * 1000 - start_time - logger.info(f'{request_id} | Total time: {elapsed:.2f}ms') + logger.info(f'Total time: {elapsed:.2f}ms') @vault_router.get( @@ -113,15 +113,15 @@ async def list_vault_entries(request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await VaultService.list_vault_entries(username, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -132,7 +132,7 @@ async def list_vault_entries(request: Request): ) finally: elapsed = time.time() * 1000 - start_time - logger.info(f'{request_id} | Total time: {elapsed:.2f}ms') + logger.info(f'Total time: {elapsed:.2f}ms') @vault_router.get( @@ -169,15 +169,15 @@ async def get_vault_entry(key_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await VaultService.get_vault_entry(username, key_name, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -188,7 +188,7 @@ async def get_vault_entry(key_name: str, request: Request): ) finally: elapsed = time.time() * 1000 - start_time - logger.info(f'{request_id} | Total time: {elapsed:.2f}ms') + logger.info(f'Total time: {elapsed:.2f}ms') @vault_router.put( @@ -215,9 +215,9 @@ async def update_vault_entry(key_name: str, update_data: UpdateVaultEntryModel, payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest( await VaultService.update_vault_entry(username, key_name, update_data, request_id) @@ -225,7 +225,7 @@ async def update_vault_entry(key_name: str, update_data: UpdateVaultEntryModel, except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -236,7 +236,7 @@ async def update_vault_entry(key_name: str, update_data: UpdateVaultEntryModel, ) finally: elapsed = time.time() * 1000 - start_time - logger.info(f'{request_id} | Total time: {elapsed:.2f}ms') + logger.info(f'Total time: {elapsed:.2f}ms') @vault_router.delete( @@ -263,15 +263,15 @@ async def delete_vault_entry(key_name: str, request: Request): payload = await auth_required(request) username = payload.get('sub') logger.info( - f'{request_id} | Username: {username} | From: {request.client.host}:{request.client.port}' + f'Username: {username} | From: {request.client.host}:{request.client.port}' ) - logger.info(f'{request_id} | Endpoint: {request.method} {str(request.url.path)}') + logger.info(f'Endpoint: {request.method} {str(request.url.path)}') return respond_rest(await VaultService.delete_vault_entry(username, key_name, request_id)) except HTTPException as e: raise e except Exception as e: - logger.critical(f'{request_id} | Unexpected error: {str(e)}', exc_info=True) + logger.critical(f'Unexpected error: {str(e)}', exc_info=True) return process_response( ResponseModel( status_code=500, @@ -282,4 +282,4 @@ async def delete_vault_entry(key_name: str, request: Request): ) finally: elapsed = time.time() * 1000 - start_time - logger.info(f'{request_id} | Total time: {elapsed:.2f}ms') + logger.info(f'Total time: {elapsed:.2f}ms') diff --git a/backend-services/routes/wsdl_routes.py b/backend-services/routes/wsdl_routes.py index 6dc51ded..9bc47dd0 100644 --- a/backend-services/routes/wsdl_routes.py +++ b/backend-services/routes/wsdl_routes.py @@ -107,7 +107,7 @@ async def get_wsdl(api_name: str, api_version: str, request: Request): return respond_rest(None, None, {'wsdl': wsdl_content, 'cached': False}, request_id, 200, start_time) except Exception as e: - logger.error(f'{request_id} | Error getting WSDL: {e}', exc_info=True) + logger.error(f'Error getting WSDL: {e}', exc_info=True) return respond_rest('WSDL999', str(e), None, None, 500, start_time) @@ -182,7 +182,7 @@ async def refresh_wsdl(api_name: str, api_version: str, request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error refreshing WSDL: {e}', exc_info=True) + logger.error(f'Error refreshing WSDL: {e}', exc_info=True) return respond_rest('WSDL999', str(e), None, None, 500, start_time) @@ -285,7 +285,7 @@ async def import_wsdl_operations(api_name: str, api_version: str, request: Reque ) except Exception as e: - logger.error(f'{request_id} | Error importing WSDL: {e}', exc_info=True) + logger.error(f'Error importing WSDL: {e}', exc_info=True) return respond_rest('WSDL999', str(e), None, None, 500, start_time) @@ -345,5 +345,5 @@ async def parse_wsdl_content(request: Request): ) except Exception as e: - logger.error(f'{request_id} | Error parsing WSDL: {e}', exc_info=True) + logger.error(f'Error parsing WSDL: {e}', exc_info=True) return respond_rest('WSDL999', str(e), None, None, 500, start_time) diff --git a/backend-services/services/endpoint_service.py b/backend-services/services/endpoint_service.py index 097c6c12..87d1711d 100644 --- a/backend-services/services/endpoint_service.py +++ b/backend-services/services/endpoint_service.py @@ -35,7 +35,9 @@ async def create_endpoint(data: CreateEndpointModel, request_id): + ' ' + data.endpoint_uri ) - cache_key = f'/{data.endpoint_method}/{data.api_name}/{data.api_version}/{data.endpoint_uri}'.replace( + # Use client_uri for routing if provided, otherwise use endpoint_uri + routing_uri = data.client_uri if data.client_uri else data.endpoint_uri + cache_key = f'/{data.endpoint_method}/{data.api_name}/{data.api_version}/{routing_uri}'.replace( '//', '/' ) if doorman_cache.get_cache('endpoint_cache', cache_key) or endpoint_collection.find_one( @@ -53,6 +55,27 @@ async def create_endpoint(data: CreateEndpointModel, request_id): error_code='END001', error_message='Endpoint already exists for the requested API name, version and URI', ).dict() + # Check if client_uri conflicts with another endpoint's client_uri or endpoint_uri + if data.client_uri: + conflict = endpoint_collection.find_one( + { + 'endpoint_method': data.endpoint_method, + 'api_name': data.api_name, + 'api_version': data.api_version, + '$or': [ + {'client_uri': data.client_uri}, + {'endpoint_uri': data.client_uri, 'client_uri': None} + ] + } + ) + if conflict: + logger.error(request_id + ' | Endpoint creation failed - client_uri conflict') + return ResponseModel( + status_code=400, + response_headers={'request_id': request_id}, + error_code='END001', + error_message='Client URI conflicts with an existing endpoint', + ).dict() # Resolve API ID from cache using canonical key "/{name}/{version}" data.api_id = doorman_cache.get_cache( 'api_id_cache', f'/{data.api_name}/{data.api_version}' @@ -115,11 +138,11 @@ async def create_endpoint(data: CreateEndpointModel, request_id): # Validate path stays within proto directory proto_dir_resolved = proto_dir.resolve() if not str(proto_path).startswith(str(proto_dir_resolved)): - logger.error(f'{request_id} | Invalid proto path detected') + logger.error(f'Invalid proto path detected') raise ValueError('Proto path must be within proto directory') # Additional validation before file operations if not str(proto_path).startswith(str(project_root.resolve())): - logger.error(f'{request_id} | Proto path outside project root') + logger.error(f'Proto path outside project root') raise ValueError('Proto path must be within project root') if not proto_path.exists(): proto_content = ( @@ -152,7 +175,7 @@ async def create_endpoint(data: CreateEndpointModel, request_id): ) if code != 0: logger.warning( - f'{request_id} | Pre-gen gRPC stubs returned {code} for {module_base}' + f'Pre-gen gRPC stubs returned {code} for {module_base}' ) try: init_path = (generated_dir / '__init__.py').resolve() @@ -162,7 +185,7 @@ async def create_endpoint(data: CreateEndpointModel, request_id): except Exception: pass except Exception as _e: - logger.debug(f'{request_id} | Skipping pre-gen gRPC stubs: {_e}') + logger.debug(f'Skipping pre-gen gRPC stubs: {_e}') return ResponseModel( status_code=201, response_headers={'request_id': request_id}, @@ -215,6 +238,28 @@ async def update_endpoint( error_code='END006', error_message='API method, name, version and URI cannot be updated', ).dict() + # Check if client_uri update conflicts with existing endpoints + if data.client_uri and data.client_uri != endpoint.get('client_uri'): + conflict = endpoint_collection.find_one( + { + 'endpoint_method': endpoint_method, + 'api_name': api_name, + 'api_version': api_version, + 'endpoint_uri': {'$ne': endpoint_uri}, # Exclude current endpoint + '$or': [ + {'client_uri': data.client_uri}, + {'endpoint_uri': data.client_uri, 'client_uri': None} + ] + } + ) + if conflict: + logger.error(request_id + ' | Endpoint update failed - client_uri conflict') + return ResponseModel( + status_code=400, + response_headers={'request_id': request_id}, + error_code='END006', + error_message='Client URI conflicts with an existing endpoint', + ).dict() not_null_data = {k: v for k, v in data.dict().items() if v is not None} if not_null_data: update_result = endpoint_collection.update_one( diff --git a/backend-services/services/gateway_service.py b/backend-services/services/gateway_service.py index 785829c7..2a05598b 100644 --- a/backend-services/services/gateway_service.py +++ b/backend-services/services/gateway_service.py @@ -164,7 +164,7 @@ async def aclose_http_client(cls) -> None: cls._http_client = None def error_response(request_id, code, message, status=404): - logger.error(f'{request_id} | REST gateway failed with code {code}') + logger.error(f'REST gateway failed with code {code}') return ResponseModel( status_code=status, response_headers={'request_id': request_id}, @@ -322,7 +322,7 @@ async def _resolve_api_from_path(path: str, request_id: str): api = await api_util.get_api(api_key, api_name_version) return api, api_name_version, endpoint_uri except Exception as e: - logger.error(f'{request_id} | API resolution failed: {str(e)}') + logger.error(f'API resolution failed: {str(e)}') return None, None, None @staticmethod @@ -345,7 +345,7 @@ async def _check_and_deduct_credits(api: dict, username: str, request_id: str): if api.get('api_credits_enabled') and username and not bool(api.get('api_public')): if not await credit_util.deduct_credit(api.get('api_credit_group'), username): - logger.warning(f'{request_id} | Credit deduction failed for user {username}') + logger.warning(f'Credit deduction failed for user {username}') return GatewayService.error_response( request_id, 'GTW008', 'User does not have any credits', status=401 ) @@ -509,7 +509,7 @@ async def rest_gateway( """ External gateway. """ - logger.info(f'{request_id} | REST gateway trying resource: {path}') + logger.info(f'REST gateway trying resource: {path}') current_time = backend_end_time = None api = None api_name_version = '' @@ -539,7 +539,7 @@ async def rest_gateway( ) try: logger.debug( - f"{request_id} | REST resolve: path={path} api_name_version={api_name_version} key1={key1} key2={key2} api_cache={'hit' if api else 'miss'} api_id_key={'set' if api_key else 'none'}" + f"REST resolve: path={path} api_name_version={api_name_version} key1={key1} key2={key2} api_cache={'hit' if api else 'miss'} api_id_key={'set' if api_key else 'none'}" ) except Exception: pass @@ -567,10 +567,24 @@ async def rest_gateway( if not any( re.fullmatch(regex_pattern.sub(r'([^/]+)', ep), composite) for ep in endpoints ): - logger.error(f'{request_id} | {endpoints} | REST gateway failed with code GTW003 (no endpoint match for {composite})') + logger.error( + f'{request_id} | {endpoints} | REST gateway failed with code GTW003 (no endpoint match for {composite})' + ) return GatewayService.error_response( request_id, 'GTW003', 'Endpoint does not exist for the requested API' ) + + # Resolve backend endpoint URI + # Use lstrip to ensure we pass a path starting with /, matching cache keys + lookup_path = '/' + endpoint_uri.lstrip('/') + endpoint_doc = await api_util.get_endpoint(api, match_method, lookup_path) + + # If found, use the backend URI (endpoint_uri field) for routing + # Note: api_util.get_endpoint now supports looking up by client_uri or endpoint_uri + if endpoint_doc and endpoint_doc.get('endpoint_uri'): + # endpoint_uri in DB has leading slash, strip it for usage here + endpoint_uri = endpoint_doc.get('endpoint_uri').lstrip('/') + client_key = request.headers.get('client-key') server = await routing_util.pick_upstream_server( api, request.method, endpoint_uri, client_key @@ -645,7 +659,7 @@ async def rest_gateway( if user_specific_api_key: headers[ai_token_headers[0]] = user_specific_api_key content_type = request.headers.get('Content-Type', '').upper() - logger.info(f'{request_id} | REST gateway to: {url}') + logger.info(f'REST gateway to: {url}') if api and api.get('api_authorization_field_swap'): try: swap_from = api.get('api_authorization_field_swap') @@ -685,7 +699,7 @@ async def rest_gateway( body = (await request.body()).decode('utf-8') await validation_util.validate_soap_request(endpoint_id, body) except Exception as e: - logger.error(f'{request_id} | Validation error: {e}') + logger.error(f'Validation error: {e}') return GatewayService.error_response(request_id, 'GTW011', str(e), status=400) # Apply request transformations if configured @@ -697,7 +711,7 @@ async def rest_gateway( headers, {}, query_params, request_transform ) except Exception as te: - logger.warning(f'{request_id} | Request transform error: {te}') + logger.warning(f'Request transform error: {te}') client = GatewayService.get_http_client() try: @@ -746,7 +760,7 @@ async def rest_gateway( {}, body, {}, request_transform ) except Exception as te: - logger.warning(f'{request_id} | Request body transform error: {te}') + logger.warning(f'Request body transform error: {te}') http_response = await request_with_resilience( client, method, @@ -806,7 +820,7 @@ async def rest_gateway( try: response_content = http_response.json() except Exception as _e: - logger.error(f'{request_id} | REST upstream malformed JSON: {str(_e)}') + logger.error(f'REST upstream malformed JSON: {str(_e)}') return ResponseModel( status_code=500, response_headers={'request_id': request_id}, @@ -820,7 +834,7 @@ async def rest_gateway( return GatewayService.error_response( request_id, 'GTW005', 'Endpoint does not exist in backend service' ) - logger.info(f'{request_id} | REST gateway status code: {http_response.status_code}') + logger.info(f'REST gateway status code: {http_response.status_code}') response_headers = {'request_id': request_id} # Response headers remain governed by explicit API allow-list. # We intentionally do NOT add SOAP defaults here to avoid exposing @@ -853,7 +867,7 @@ async def rest_gateway( dict(response_headers), response_content, http_response.status_code, response_transform ) except Exception as te: - logger.warning(f'{request_id} | Response transform error: {te}') + logger.warning(f'Response transform error: {te}') return ResponseModel( status_code=final_status, @@ -881,22 +895,22 @@ async def rest_gateway( error_message='Gateway timeout', ).dict() except Exception: - logger.error(f'{request_id} | REST gateway failed with code GTW006', exc_info=True) + logger.error(f'REST gateway failed with code GTW006', exc_info=True) return GatewayService.error_response( request_id, 'GTW006', 'Internal server error', status=500 ) finally: if current_time: - logger.info(f'{request_id} | Gateway time {current_time - start_time}ms') + logger.info(f'Gateway time {current_time - start_time}ms') if backend_end_time and current_time: - logger.info(f'{request_id} | Backend time {backend_end_time - current_time}ms') + logger.info(f'Backend time {backend_end_time - current_time}ms') @staticmethod async def soap_gateway(username, request, request_id, start_time, path, url=None, retry=0): """ External SOAP gateway. """ - logger.info(f'{request_id} | SOAP gateway trying resource: {path}') + logger.info(f'SOAP gateway trying resource: {path}') current_time = backend_end_time = None api = None api_name_version = '' @@ -925,7 +939,7 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None ) try: logger.debug( - f"{request_id} | SOAP resolve: path={path} api_name_version={api_name_version} key1={key1} key2={key2} api_cache={'hit' if api else 'miss'} api_id_key={'set' if api_key else 'none'}" + f"SOAP resolve: path={path} api_name_version={api_name_version} key1={key1} key2={key2} api_cache={'hit' if api else 'miss'} api_id_key={'set' if api_key else 'none'}" ) except Exception: pass @@ -949,7 +963,7 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None return await CrudService.handle_soap(api, request, request_id, body=request._body if hasattr(request, '_body') else await request.body()) endpoints = await api_util.get_api_endpoints(api.get('api_id')) - logger.info(f'{request_id} | SOAP gateway endpoints: {endpoints}') + logger.info(f'SOAP gateway endpoints: {endpoints}') if not endpoints: return GatewayService.error_response( request_id, 'GTW002', 'No endpoints found for the requested API' @@ -972,8 +986,6 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None ) url = server.rstrip('/') + '/' + endpoint_uri.lstrip('/') logger.info(f'{request_id} | SOAP gateway to: {url}') - - retry = api.get('api_allowed_retry_count') or 0 if api.get('api_credits_enabled') and username and not bool(api.get('api_public')): if not await credit_util.deduct_credit(api.get('api_credit_group'), username): @@ -1071,9 +1083,9 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None timestamp_ttl_seconds=ws_security_config.get('timestamp_ttl_seconds', 300), ) envelope = inject_ws_security(envelope, security_header) - logger.debug(f'{request_id} | WS-Security header injected') + logger.debug(f'WS-Security header injected') except Exception as wsse: - logger.warning(f'{request_id} | WS-Security injection failed: {wsse}') + logger.warning(f'WS-Security injection failed: {wsse}') if api and api.get('api_authorization_field_swap'): try: swap_from = api.get('api_authorization_field_swap') @@ -1107,7 +1119,7 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None if endpoint_id: await validation_util.validate_soap_request(endpoint_id, envelope) except Exception as e: - logger.error(f'{request_id} | Validation error: {e}') + logger.error(f'Validation error: {e}') return GatewayService.error_response(request_id, 'GTW011', str(e), status=400) client = GatewayService.get_http_client() try: @@ -1138,7 +1150,7 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None return GatewayService.error_response( request_id, 'GTW005', 'Endpoint does not exist in backend service' ) - logger.info(f'{request_id} | SOAP gateway status code: {http_response.status_code}') + logger.info(f'SOAP gateway status code: {http_response.status_code}') response_headers = {'request_id': request_id} # Only expose upstream response headers explicitly allowed by API allowed_lower = {h.lower() for h in (api_allowed or [])} @@ -1185,19 +1197,19 @@ async def soap_gateway(username, request, request_id, start_time, path, url=None error_message='Gateway timeout', ).dict() except Exception: - logger.error(f'{request_id} | SOAP gateway failed with code GTW006') + logger.error(f'SOAP gateway failed with code GTW006') return GatewayService.error_response( request_id, 'GTW006', 'Internal server error', status=500 ) finally: if current_time: - logger.info(f'{request_id} | Gateway time {current_time - start_time}ms') + logger.info(f'Gateway time {current_time - start_time}ms') if backend_end_time and current_time: - logger.info(f'{request_id} | Backend time {backend_end_time - current_time}ms') + logger.info(f'Backend time {backend_end_time - current_time}ms') @staticmethod async def graphql_gateway(username, request, request_id, start_time, path, url=None, retry=0): - logger.info(f'{request_id} | GraphQL gateway processing request') + logger.info(f'GraphQL gateway processing request') current_time = backend_end_time = None try: if not url: @@ -1208,7 +1220,7 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N if not api: api = await api_util.get_api(None, api_path) if not api: - logger.error(f'{request_id} | API not found: {api_path}') + logger.error(f'API not found: {api_path}') return GatewayService.error_response( request_id, 'GTW001', f'API does not exist: {api_path}' ) @@ -1275,7 +1287,7 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N from utils.graphql_util import validate_query_depth is_valid, depth_error = validate_query_depth(query, max_depth) if not is_valid: - logger.warning(f'{request_id} | GraphQL depth limit exceeded: {depth_error}') + logger.warning(f'GraphQL depth limit exceeded: {depth_error}') return GatewayService.error_response(request_id, 'GTW013', depth_error, status=400) try: @@ -1296,7 +1308,7 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N client_key = request.headers.get('client-key') server = await routing_util.pick_upstream_server(api, 'POST', '/graphql', client_key) if not server: - logger.error(f'{request_id} | No upstream servers configured for {api_path}') + logger.error(f'No upstream servers configured for {api_path}') return GatewayService.error_response( request_id, 'GTW001', 'No upstream servers configured' ) @@ -1322,7 +1334,7 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N result = await session.execute(gql(query), variable_values=variables) # type: ignore except Exception as _e: logger.debug( - f'{request_id} | GraphQL Client execution failed; falling back to HTTP: {_e}' + f'GraphQL Client execution failed; falling back to HTTP: {_e}' ) # Fallback to HTTP POST @@ -1367,7 +1379,7 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N result = data backend_end_time = time.time() * 1000 - logger.info(f'{request_id} | GraphQL gateway status code: 200') + logger.info(f'GraphQL gateway status code: 200') response_headers = {'request_id': request_id} allowed_lower = {h.lower() for h in (allowed_headers or [])} for key, value in headers.items(): @@ -1411,14 +1423,14 @@ async def graphql_gateway(username, request, request_id, start_time, path, url=N error_message='Gateway timeout', ).dict() except Exception as e: - logger.error(f'{request_id} | GraphQL gateway failed with code GTW006: {str(e)}') + logger.error(f'GraphQL gateway failed with code GTW006: {str(e)}') error_msg = str(e)[:255] if len(str(e)) > 255 else str(e) return GatewayService.error_response(request_id, 'GTW006', error_msg, status=500) finally: if current_time: - logger.info(f'{request_id} | Gateway time {current_time - start_time}ms') + logger.info(f'Gateway time {current_time - start_time}ms') if backend_end_time and current_time: - logger.info(f'{request_id} | Backend time {backend_end_time - current_time}ms') + logger.info(f'Backend time {backend_end_time - current_time}ms') @staticmethod async def grpc_gateway( @@ -1432,14 +1444,14 @@ async def grpc_gateway( if api_name is None: path_parts = path.strip('/').split('/') if len(path_parts) < 1: - logger.error(f'{request_id} | Invalid API path format: {path}') + logger.error(f'Invalid API path format: {path}') return GatewayService.error_response( request_id, 'GTW001', 'Invalid API path format', status=404 ) api_name = path_parts[-1] api_version = request.headers.get('X-API-Version', 'v1') api_path = f'{api_name}/{api_version}' - logger.info(f'{request_id} | Processing gRPC request for API: {api_path}') + logger.info(f'Processing gRPC request for API: {api_path}') # Internal CRUD Hook (Early) api_check = doorman_cache.get_cache('api_cache', api_path) @@ -1462,7 +1474,7 @@ async def grpc_gateway( request_id, 'GTW011', 'Invalid request body format', status=400 ) except json.JSONDecodeError: - logger.error(f'{request_id} | Invalid JSON in request body') + logger.error(f'Invalid JSON in request body') return GatewayService.error_response( request_id, 'GTW011', 'Invalid JSON in request body', status=400 ) @@ -1527,7 +1539,7 @@ async def grpc_gateway( module_base = api_pkg or pkg_override_valid or default_base try: logger.info( - f'{request_id} | gRPC module_base resolved: module_base={module_base} ' + f'gRPC module_base resolved: module_base={module_base} ' f'api_pkg={api_pkg_raw!r} pkg_override={pkg_override_raw!r} default_base={default_base}' ) except Exception: @@ -1587,7 +1599,7 @@ async def grpc_gateway( sys.path.insert(0, gen_dir_str) try: logger.info( - f'{request_id} | sys.path updated for gRPC import. project_root={proj_root_str}, generated_dir={gen_dir_str}' + f'sys.path updated for gRPC import. project_root={proj_root_str}, generated_dir={gen_dir_str}' ) except Exception: pass @@ -1606,16 +1618,16 @@ async def grpc_gateway( pb2 = importlib.import_module(gen_pb2_name) pb2_grpc = importlib.import_module(gen_pb2_grpc_name) logger.info( - f'{request_id} | Successfully imported gRPC modules: {pb2.__name__} and {pb2_grpc.__name__}' + f'Successfully imported gRPC modules: {pb2.__name__} and {pb2_grpc.__name__}' ) except ModuleNotFoundError as mnf_exc: importlib.invalidate_caches() logger.warning( - f'{request_id} | gRPC modules not found, will attempt proto generation: {str(mnf_exc)}' + f'gRPC modules not found, will attempt proto generation: {str(mnf_exc)}' ) except ImportError as imp_exc: logger.error( - f'{request_id} | ImportError loading gRPC modules (likely broken import in generated file): {str(imp_exc)}' + f'ImportError loading gRPC modules (likely broken import in generated file): {str(imp_exc)}' ) mod_pb2 = f'{module_base}_pb2' mod_pb2_grpc = f'{module_base}_pb2_grpc' @@ -1631,7 +1643,7 @@ async def grpc_gateway( ) except Exception as import_exc: logger.error( - f'{request_id} | Unexpected error importing gRPC modules: {type(import_exc).__name__}: {str(import_exc)}' + f'Unexpected error importing gRPC modules: {type(import_exc).__name__}: {str(import_exc)}' ) return GatewayService.error_response( request_id, @@ -1647,7 +1659,7 @@ async def grpc_gateway( pb2_check_path = (generated_dir / (module_base + '_pb2.py')).resolve() if GatewayService._validate_under_base(generated_dir, pb2_check_path): logger.info( - f'{request_id} | gRPC generated check: proto_path={proto_path} exists={proto_path.exists()} generated_dir={generated_dir} pb2={module_base}_pb2.py={pb2_check_path.exists()}' + f'gRPC generated check: proto_path={proto_path} exists={proto_path.exists()} generated_dir={generated_dir} pb2={module_base}_pb2.py={pb2_check_path.exists()}' ) except Exception: pass @@ -1704,7 +1716,7 @@ async def grpc_gateway( '"""Generated gRPC code."""\n', encoding='utf-8' ) except Exception as ge: - logger.error(f'{request_id} | On-demand proto generation failed: {ge}') + logger.error(f'On-demand proto generation failed: {ge}') try: import sys as _sys _is_pytest = 'PYTEST_CURRENT_TEST' in os.environ or 'pytest' in _sys.modules @@ -1722,7 +1734,7 @@ async def grpc_gateway( ) except Exception as ge: logger.error( - f'{request_id} | Proto file not found and generation skipped: {ge}' + f'Proto file not found and generation skipped: {ge}' ) try: import sys as _sys @@ -1740,7 +1752,7 @@ async def grpc_gateway( if not api: api = await api_util.get_api(None, api_path) if not api: - logger.error(f'{request_id} | API not found: {api_path}') + logger.error(f'API not found: {api_path}') return GatewayService.error_response( request_id, 'GTW001', f'API does not exist: {api_path}', status=404 ) @@ -1748,7 +1760,7 @@ async def grpc_gateway( client_key = request.headers.get('client-key') server = await routing_util.pick_upstream_server(api, 'POST', '/grpc', client_key) if not server: - logger.error(f'{request_id} | No upstream servers configured for {api_path}') + logger.error(f'No upstream servers configured for {api_path}') return GatewayService.error_response( request_id, 'GTW001', 'No upstream servers configured', status=404 ) @@ -1851,22 +1863,22 @@ async def grpc_gateway( try: body = await request.json() if not isinstance(body, dict): - logger.error(f'{request_id} | Invalid request body format') + logger.error(f'Invalid request body format') return GatewayService.error_response( request_id, 'GTW011', 'Invalid request body format', status=400 ) except json.JSONDecodeError: - logger.error(f'{request_id} | Invalid JSON in request body') + logger.error(f'Invalid JSON in request body') return GatewayService.error_response( request_id, 'GTW011', 'Invalid JSON in request body', status=400 ) if 'method' not in body: - logger.error(f'{request_id} | Missing method in request body') + logger.error(f'Missing method in request body') return GatewayService.error_response( request_id, 'GTW011', 'Missing method in request body', status=400 ) if 'message' not in body: - logger.error(f'{request_id} | Missing message in request body') + logger.error(f'Missing message in request body') return GatewayService.error_response( request_id, 'GTW011', 'Missing message in request body', status=400 ) @@ -1942,7 +1954,7 @@ async def grpc_gateway( sys.path.insert(0, gen_dir_str) try: logger.info( - f'{request_id} | sys.path prepared for import: project_root={proj_root_str}, generated_dir={gen_dir_str}' + f'sys.path prepared for import: project_root={proj_root_str}, generated_dir={gen_dir_str}' ) except Exception: pass @@ -1953,7 +1965,7 @@ async def grpc_gateway( if use_imported: pb2_module = pb2 service_module = pb2_grpc - logger.info(f'{request_id} | Using imported gRPC modules for {module_name}') + logger.info(f'Using imported gRPC modules for {module_name}') else: if not proto_path.exists(): try: @@ -1967,7 +1979,7 @@ async def grpc_gateway( service_module = importlib.import_module(f'{module_name}_pb2_grpc') use_imported = True except Exception: - logger.error(f'{request_id} | Proto file not found: {str(proto_path)}') + logger.error(f'Proto file not found: {str(proto_path)}') return GatewayService.error_response( request_id, 'GTW012', @@ -1982,14 +1994,14 @@ async def grpc_gateway( generated_dir, pb2_path ) or not GatewayService._validate_under_base(generated_dir, pb2_grpc_path): logger.error( - f'{request_id} | Invalid path for generated modules: pb2={pb2_path} pb2_grpc={pb2_grpc_path}' + f'Invalid path for generated modules: pb2={pb2_path} pb2_grpc={pb2_grpc_path}' ) return GatewayService.error_response( request_id, 'GTW012', 'Invalid generated module path', status=400 ) if not (pb2_path.is_file() and pb2_grpc_path.is_file()): logger.error( - f"{request_id} | Generated modules not found for '{module_name}' pb2={pb2_path} exists={pb2_path.is_file()} pb2_grpc={pb2_grpc_path} exists={pb2_grpc_path.is_file()}" + f"Generated modules not found for '{module_name}' pb2={pb2_path} exists={pb2_path.is_file()} pb2_grpc={pb2_grpc_path} exists={pb2_grpc_path.is_file()}" ) if isinstance(url, str) and url.startswith(('http://', 'https://')): try: @@ -2047,7 +2059,7 @@ async def grpc_gateway( import_name_pb2 = f'{module_name}_pb2' import_name_grpc = f'{module_name}_pb2_grpc' logger.info( - f'{request_id} | Importing generated modules: {import_name_pb2} and {import_name_grpc}' + f'Importing generated modules: {import_name_pb2} and {import_name_grpc}' ) try: pb2_module = importlib.import_module(import_name_pb2) @@ -2056,13 +2068,13 @@ async def grpc_gateway( alt_pb2 = f'generated.{module_name}_pb2' alt_grpc = f'generated.{module_name}_pb2_grpc' logger.info( - f'{request_id} | Retrying import via generated package: {alt_pb2} and {alt_grpc}' + f'Retrying import via generated package: {alt_pb2} and {alt_grpc}' ) pb2_module = importlib.import_module(alt_pb2) service_module = importlib.import_module(alt_grpc) except ImportError as e: logger.error( - f'{request_id} | Failed to import gRPC module: {str(e)}', exc_info=True + f'Failed to import gRPC module: {str(e)}', exc_info=True ) if isinstance(url, str) and url.startswith(('http://', 'https://')): try: @@ -2153,7 +2165,7 @@ async def grpc_gateway( ), ).dict() - logger.info(f'{request_id} | Connecting to gRPC upstream: {url}') + logger.info(f'Connecting to gRPC upstream: {url}') # Create appropriate gRPC channel depending on scheme if 'grpc_target' in locals() and use_tls: try: @@ -2179,7 +2191,7 @@ async def grpc_gateway( try: from google.protobuf import descriptor_pool, message_factory except Exception as e: - logger.debug(f'{request_id} | protobuf descriptor imports failed: {e}') + logger.debug(f'protobuf descriptor imports failed: {e}') descriptor_pool = None message_factory = None @@ -2196,7 +2208,7 @@ async def grpc_gateway( request_class = mf.GetPrototype(method_desc.input_type) # type: ignore[arg-type] reply_class = mf.GetPrototype(method_desc.output_type) # type: ignore[arg-type] except Exception as de: - logger.debug(f'{request_id} | Descriptor-based resolution failed: {de}') + logger.debug(f'Descriptor-based resolution failed: {de}') # Reflection fallback if enabled and not yet resolved if ( @@ -2237,10 +2249,10 @@ async def _single_req(): request_class = mf.GetPrototype(method_desc.input_type) reply_class = mf.GetPrototype(method_desc.output_type) except Exception as re: - logger.debug(f'{request_id} | Reflection resolution failed: {re}') + logger.debug(f'Reflection resolution failed: {re}') break except Exception as re: - logger.debug(f'{request_id} | Reflection import/use failed: {re}') + logger.debug(f'Reflection import/use failed: {re}') # Legacy fallback: assume MethodRequest/MethodReply classes in pb2_module # Special-case common Empty method to use google.protobuf.Empty @@ -2273,7 +2285,7 @@ async def _single_req(): reply_class = reply_class or getattr(pb2_module, reply_class_name) except AttributeError as attr_err: logger.error( - f'{request_id} | Message types {request_class_name}/{reply_class_name} not found: {attr_err}' + f'Message types {request_class_name}/{reply_class_name} not found: {attr_err}' ) return GatewayService.error_response( request_id, @@ -2287,7 +2299,7 @@ async def _single_req(): request_message = request_class() except Exception as create_err: logger.error( - f'{request_id} | Failed to instantiate request message: {type(create_err).__name__}: {str(create_err)}' + f'Failed to instantiate request message: {type(create_err).__name__}: {str(create_err)}' ) return GatewayService.error_response( request_id, @@ -2297,7 +2309,7 @@ async def _single_req(): ) except Exception as e: logger.error( - f'{request_id} | Unexpected error in message type resolution: {type(e).__name__}: {str(e)}' + f'Unexpected error in message type resolution: {type(e).__name__}: {str(e)}' ) return GatewayService.error_response( request_id, @@ -2351,7 +2363,7 @@ async def _single_req(): got_response = False try: logger.info( - f'{request_id} | gRPC entering attempts={attempts} stream_mode={stream_mode or "unary"} method={service_name}.{method_name}' + f'gRPC entering attempts={attempts} stream_mode={stream_mode or "unary"} method={service_name}.{method_name}' ) except Exception: pass @@ -2360,7 +2372,7 @@ async def _single_req(): full_method = f'/{module_base}.{service_name}/{method_name}' try: logger.info( - f'{request_id} | gRPC attempt={attempt + 1}/{attempts} calling {full_method}' + f'gRPC attempt={attempt + 1}/{attempts} calling {full_method}' ) except Exception: pass @@ -2525,7 +2537,7 @@ async def _gen_bidi(): last_exc = None try: logger.info( - f'{request_id} | gRPC unary success; stream_mode={stream_mode or "unary"}' + f'gRPC unary success; stream_mode={stream_mode or "unary"}' ) except Exception: pass @@ -2535,9 +2547,9 @@ async def _gen_bidi(): try: code = getattr(e2, 'code', lambda: None)() cname = str(getattr(code, 'name', '') or 'UNKNOWN') - logger.info(f'{request_id} | gRPC primary call raised: {cname}') + logger.info(f'gRPC primary call raised: {cname}') except Exception: - logger.info(f'{request_id} | gRPC primary call raised non-grpc exception') + logger.info(f'gRPC primary call raised non-grpc exception') final_code_name = str(code.name) if getattr(code, 'name', None) else 'ERROR' if attempt < attempts - 1 and is_idempotent and code in retryable: retries_made += 1 @@ -2713,9 +2725,9 @@ async def _gen_bidi_alt(): try: code3 = getattr(e3, 'code', lambda: None)() cname3 = str(getattr(code3, 'name', '') or 'UNKNOWN') - logger.info(f'{request_id} | gRPC alt call raised: {cname3}') + logger.info(f'gRPC alt call raised: {cname3}') except Exception: - logger.info(f'{request_id} | gRPC alt call raised non-grpc exception') + logger.info(f'gRPC alt call raised non-grpc exception') final_code_name = ( str(code3.name) if getattr(code3, 'name', None) else 'ERROR' ) @@ -2734,12 +2746,12 @@ async def _gen_bidi_alt(): code_obj = getattr(last_exc, 'code', lambda: None)() if code_obj and hasattr(code_obj, 'name'): code_name = str(code_obj.name).upper() - logger.info(f'{request_id} | gRPC call failed with status: {code_name}') + logger.info(f'gRPC call failed with status: {code_name}') else: - logger.warning(f'{request_id} | gRPC exception has no valid status code') + logger.warning(f'gRPC exception has no valid status code') except Exception as code_extract_err: logger.warning( - f'{request_id} | Failed to extract gRPC status code: {str(code_extract_err)}' + f'Failed to extract gRPC status code: {str(code_extract_err)}' ) status_map = { @@ -2777,7 +2789,7 @@ async def _gen_bidi_alt(): details = f'gRPC error: {code_name}' logger.error( - f'{request_id} | gRPC call failed after {retries_made} retries. ' + f'gRPC call failed after {retries_made} retries. ' f'Status: {code_name}, HTTP: {http_status}, Details: {details[:100]}' ) @@ -2799,7 +2811,7 @@ async def _gen_bidi_alt(): if not got_response and last_exc is None: try: logger.error( - f'{request_id} | gRPC loop ended with no response and no exception; returning 500 UNKNOWN' + f'gRPC loop ended with no response and no exception; returning 500 UNKNOWN' ) except Exception: pass @@ -2829,7 +2841,7 @@ async def _gen_bidi_alt(): ) except RecursionError as re: logger.error( - f'{request_id} | Protobuf recursion limit exceeded for field {field.name}: {str(re)}' + f'Protobuf recursion limit exceeded for field {field.name}: {str(re)}' ) return GatewayService.error_response( request_id, @@ -2839,7 +2851,7 @@ async def _gen_bidi_alt(): ) except Exception as e: logger.error( - f'{request_id} | Failed to convert protobuf field {field.name}: {str(e)}' + f'Failed to convert protobuf field {field.name}: {str(e)}' ) response_dict[field.name] = str(value) else: @@ -2859,7 +2871,7 @@ async def _gen_bidi_alt(): pass try: logger.info( - f'{request_id} | gRPC return 200 with items={bool(response_dict.get("items"))}' + f'gRPC return 200 with items={bool(response_dict.get("items"))}' ) except Exception: pass @@ -2916,7 +2928,7 @@ async def _gen_bidi_alt(): pass logger.error( - f'{request_id} | gRPC gateway exception in outer handler. ' + f'gRPC gateway exception in outer handler. ' f'Type: {type(e).__name__}, Status: {code_name}, HTTP: {http_status}, Error: {str(e)[:200]}' ) @@ -2932,12 +2944,12 @@ async def _gen_bidi_alt(): ).dict() finally: if current_time: - logger.info(f'{request_id} | Gateway time {current_time - start_time}ms') + logger.info(f'Gateway time {current_time - start_time}ms') if backend_end_time and current_time: - logger.info(f'{request_id} | Backend time {backend_end_time - current_time}ms') + logger.info(f'Backend time {backend_end_time - current_time}ms') async def _make_graphql_request( - self, url: str, query: str, headers: dict[str, str] = None + self, url: str, query: str, request_id: str, headers: dict[str, str] = None ) -> dict: try: if headers is None: diff --git a/backend-services/services/logging_service.py b/backend-services/services/logging_service.py index 6fa11acf..caab93e7 100644 --- a/backend-services/services/logging_service.py +++ b/backend-services/services/logging_service.py @@ -60,6 +60,8 @@ async def get_logs( limit: int = 100, offset: int = 0, request_id_param: str = None, + type: str | None = None, + exclude_type: str | None = None, ) -> dict[str, Any]: """ Retrieve and filter logs based on various criteria @@ -81,38 +83,15 @@ async def get_logs( logs = [] total_count = 0 - # Case A: Use in-memory buffer when it has content; it's fastest and covers both envs - if buffer_lines: - for line in reversed(buffer_lines): - log_entry = self._parse_log_line(line) - if log_entry and self._matches_filters( - log_entry, - { - 'start_date': start_date, - 'end_date': end_date, - 'start_time': start_time, - 'end_time': end_time, - 'user': user, - 'api': api, - 'endpoint': endpoint, - 'request_id': request_id, - 'method': method, - 'ip_address': ip_address, - 'min_response_time': min_response_time, - 'max_response_time': max_response_time, - 'level': level, - }, - ): - total_count += 1 - if total_count > offset and len(logs) < limit: - logs.append(log_entry) - if len(logs) >= limit: - break - logs.reverse() - elif log_files: + # Strategy: Prioritize log files (history), fallback to in-memory buffer + + # Container for all matching logs + all_logs = [] + + if log_files: log_files.sort(key=lambda x: os.path.getmtime(x), reverse=True) - # Cap file scanning to a few recent files for performance - log_files = log_files[:3] + # Scan more files to ensure we don't miss logs + log_files = log_files[:20] for log_file in log_files: if not os.path.exists(log_file): @@ -138,29 +117,53 @@ async def get_logs( 'min_response_time': min_response_time, 'max_response_time': max_response_time, 'level': level, + 'type': type, + 'exclude_type': exclude_type, }, ): - total_count += 1 - if len(logs) < limit and total_count > offset: - logs.append(log_entry) - - if len(logs) >= limit: - break - - if len(logs) >= limit: - break + all_logs.append(log_entry) except Exception as e: - logger.warning(f'Error reading log file {log_file}: {str(e)}') + logger.warning(f'{request_id_param} | Error reading log file {log_file}: {str(e)}') continue - else: - # Nothing to read - pass + + # ALWAYS check buffer for recent logs, regardless of files + if buffer_lines: + for line in reversed(buffer_lines): + log_entry = self._parse_log_line(line) + if log_entry and self._matches_filters( + log_entry, + { + 'start_date': start_date, + 'end_date': end_date, + 'start_time': start_time, + 'end_time': end_time, + 'user': user, + 'api': api, + 'endpoint': endpoint, + 'request_id': request_id, + 'method': method, + 'ip_address': ip_address, + 'min_response_time': min_response_time, + 'max_response_time': max_response_time, + 'level': level, + 'type': type, + 'exclude_type': exclude_type, + }, + ): + all_logs.append(log_entry) + + # Sort by timestamp descending (newest first) + all_logs.sort(key=lambda x: x.get('timestamp', ''), reverse=True) + total_count = len(all_logs) + + # Paginate: skip offset, take limit + logs = all_logs[offset:offset + limit] return {'logs': logs, 'total': total_count, 'has_more': total_count > offset + limit} except Exception as e: - logger.error(f'Error retrieving logs: {str(e)}', exc_info=True) + logger.error(f'{request_id_param} | Error retrieving logs: {str(e)}', exc_info=True) raise HTTPException(status_code=500, detail='Failed to retrieve logs') def get_available_log_files(self) -> list[str]: @@ -324,6 +327,8 @@ def _parse_log_line(self, line: str) -> dict[str, Any] | None: name = rec.get('name', '') level = rec.get('level', '') structured = self._extract_structured_data(message) + if rec.get('request_id'): + structured['request_id'] = rec.get('request_id') return { 'timestamp': timestamp if isinstance(timestamp, str) @@ -353,9 +358,15 @@ def _parse_log_line(self, line: str) -> dict[str, Any] | None: structured_data = self._extract_structured_data(message) if request_id: structured_data['request_id'] = request_id + + # User Requirement: any log without a request id should be a debug log + final_level = level + if not request_id and not structured_data.get('request_id'): + final_level = 'DEBUG' + return { 'timestamp': timestamp.isoformat(), - 'level': level, + 'level': final_level, 'message': message, 'source': name, **structured_data, @@ -408,6 +419,27 @@ def _extract_structured_data(self, message: str) -> dict[str, Any]: if status_match: data['status_code'] = status_match.group(1) + # Infer request type + # Default to platform + request_type = 'platform' + + endpoint_val = data.get('endpoint', '').lower() + if endpoint_val: + if any(x in endpoint_val for x in ('/rest', '/soap', '/graphql')): + request_type = 'gateway' + elif '/authorization' in endpoint_val: + request_type = 'auth' + + # Fallback to message content if endpoint not present + if request_type == 'platform': + msg_lower = message.lower() + if any(x in msg_lower for x in ('rest gateway', 'soap gateway', 'graphql gateway', 'upstream')): + request_type = 'gateway' + elif any(x in msg_lower for x in ('login', 'register', 'token', 'permission')): + request_type = 'auth' + + data['type'] = request_type + return data def _accumulate_stats_line(self, line: str, stats: dict[str, Any]) -> None: @@ -470,6 +502,12 @@ def _matches_filters(self, log_entry: dict[str, Any], filters: dict[str, Any]) - else: start_time = datetime.strptime(filters['start_time'], '%H:%M').time() start_datetime = datetime.combine(timestamp.date(), start_time) + + # Make timezone-aware if timestamp is timezone-aware + if timestamp.tzinfo is not None and start_datetime.tzinfo is None: + start_datetime = start_datetime.replace(tzinfo=timezone.utc) + elif timestamp.tzinfo is None and start_datetime.tzinfo is not None: + timestamp = timestamp.replace(tzinfo=timezone.utc) if timestamp < start_datetime: return False @@ -486,11 +524,17 @@ def _matches_filters(self, log_entry: dict[str, Any], filters: dict[str, Any]) - else: end_time = datetime.strptime(filters['end_time'], '%H:%M').time() end_datetime = datetime.combine(timestamp.date(), end_time) + + # Make timezone-aware if timestamp is timezone-aware + if timestamp.tzinfo is not None and end_datetime.tzinfo is None: + end_datetime = end_datetime.replace(tzinfo=timezone.utc) + elif timestamp.tzinfo is None and end_datetime.tzinfo is not None: + timestamp = timestamp.replace(tzinfo=timezone.utc) if timestamp > end_datetime: return False - for field in ['user', 'api', 'endpoint', 'request_id', 'method', 'ip_address', 'level']: + for field in ['user', 'api', 'endpoint', 'request_id', 'method', 'ip_address', 'level', 'type']: if filters.get(field) and filters[field].strip(): filter_value = filters[field].strip().lower() log_value = str(log_entry.get(field, '')).lower() @@ -541,6 +585,19 @@ def _matches_filters(self, log_entry: dict[str, Any], filters: dict[str, Any]) - except (ValueError, TypeError): return False + if filters.get('exclude_type') and filters['exclude_type'].strip(): + exclude_val = filters['exclude_type'].strip().lower() + log_type = str(log_entry.get('type', '')).lower() + endpoint = str(log_entry.get('endpoint', '')).lower() + + # Direct type match exclusion + if log_type == exclude_val: + return False + + # Specific logic for 'platform' exclusion: also hide known platform endpoints + if exclude_val == 'platform' and '/platform/' in endpoint: + return False + applied_filters = [f'{k}={v}' for k, v in filters.items() if v and str(v).strip()] if applied_filters: logger.debug(f'Log entry passed all filters: {", ".join(applied_filters)}') diff --git a/backend-services/services/subscription_service.py b/backend-services/services/subscription_service.py index cbdcbd5c..4148552a 100644 --- a/backend-services/services/subscription_service.py +++ b/backend-services/services/subscription_service.py @@ -59,7 +59,7 @@ async def get_user_subscriptions(username, request_id): """ Get user subscriptions. """ - logger.info(f'{request_id} | Getting subscriptions for: {username}') + logger.info(f'Getting subscriptions for: {username}') subscriptions = doorman_cache.get_cache('user_subscription_cache', username) if ( @@ -69,7 +69,7 @@ async def get_user_subscriptions(username, request_id): ): subscriptions = subscriptions_collection.find_one({'username': username}) if not subscriptions: - logger.info(f'{request_id} | No subscriptions found; returning empty list') + logger.info(f'No subscriptions found; returning empty list') return ResponseModel(status_code=200, response={'apis': []}).dict() if subscriptions.get('_id'): @@ -78,7 +78,7 @@ async def get_user_subscriptions(username, request_id): doorman_cache.set_cache('user_subscription_cache', username, subscriptions) apis = subscriptions.get('apis', []) if isinstance(subscriptions, dict) else [] - logger.info(f'{request_id} | Subscriptions retrieved successfully') + logger.info(f'Subscriptions retrieved successfully') return ResponseModel( status_code=200, response={'apis': apis, 'subscriptions': {'apis': apis}} ).dict() @@ -89,11 +89,11 @@ async def subscribe(data: SubscribeModel, request_id): Subscribe to an API. """ logger.info( - f'{request_id} | Subscribing {data.username} to API: {data.api_name}/{data.api_version}' + f'Subscribing {data.username} to API: {data.api_name}/{data.api_version}' ) api = await SubscriptionService.api_exists(data.api_name, data.api_version) if not api: - logger.error(f'{request_id} | Subscription failed with code SUB003') + logger.error(f'Subscription failed with code SUB003') return ResponseModel( status_code=404, response_headers={'request_id': request_id}, @@ -115,7 +115,7 @@ async def subscribe(data: SubscribeModel, request_id): 'apis' in user_subscriptions and f'{data.api_name}/{data.api_version}' in user_subscriptions['apis'] ): - logger.error(f'{request_id} | Subscription failed with code SUB004') + logger.error(f'Subscription failed with code SUB004') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -133,7 +133,7 @@ async def subscribe(data: SubscribeModel, request_id): del user_subscriptions['_id'] if user_subscriptions: doorman_cache.set_cache('user_subscription_cache', data.username, user_subscriptions) - logger.info(f'{request_id} | Subscription successful') + logger.info(f'Subscription successful') return ResponseModel( status_code=200, response={'message': 'Successfully subscribed to the API'} ).dict() @@ -144,7 +144,7 @@ async def unsubscribe(data: SubscribeModel, request_id): Unsubscribe from an API. """ logger.info( - f'{request_id} | Unsubscribing {data.username} from API: {data.api_name}/{data.api_version}' + f'Unsubscribing {data.username} from API: {data.api_name}/{data.api_version}' ) api = await SubscriptionService.api_exists(data.api_name, data.api_version) if not api: @@ -163,7 +163,7 @@ async def unsubscribe(data: SubscribeModel, request_id): not user_subscriptions or f'{data.api_name}/{data.api_version}' not in user_subscriptions.get('apis', []) ): - logger.error(f'{request_id} | Unsubscription failed with code SUB006') + logger.error(f'Unsubscription failed with code SUB006') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -180,7 +180,7 @@ async def unsubscribe(data: SubscribeModel, request_id): del user_subscriptions['_id'] if user_subscriptions: doorman_cache.set_cache('user_subscription_cache', data.username, user_subscriptions) - logger.info(f'{request_id} | Unsubscription successful') + logger.info(f'Unsubscription successful') return ResponseModel( status_code=200, response={'message': 'Successfully unsubscribed from the API'} ).dict() diff --git a/backend-services/services/user_service.py b/backend-services/services/user_service.py index 997c8aff..b9b6b19b 100644 --- a/backend-services/services/user_service.py +++ b/backend-services/services/user_service.py @@ -70,12 +70,12 @@ async def get_user_by_username(username: str, request_id: str) -> dict: """ Retrieve a user by username. """ - logger.info(f'{request_id} | Getting user: {username}') + logger.info(f'Getting user: {username}') user = doorman_cache.get_cache('user_cache', username) if not user: user = await db_find_one(user_collection, {'username': username}) if not user: - logger.error(f'{request_id} | User retrieval failed with code USR002') + logger.error(f'User retrieval failed with code USR002') return ResponseModel( status_code=404, error_code='USR002', error_message='User not found' ).dict() @@ -85,7 +85,7 @@ async def get_user_by_username(username: str, request_id: str) -> dict: del user['password'] doorman_cache.set_cache('user_cache', username, user) if not user: - logger.error(f'{request_id} | User retrieval failed with code USR002') + logger.error(f'User retrieval failed with code USR002') return ResponseModel( status_code=404, response_headers={'request_id': request_id}, @@ -114,7 +114,7 @@ async def get_user_by_username(username: str, request_id: str) -> dict: user['bandwidth_resets_at'] = resets_at except Exception: pass - logger.info(f'{request_id} | User retrieval successful') + logger.info(f'User retrieval successful') return ResponseModel(status_code=200, response=user).dict() @staticmethod @@ -122,10 +122,10 @@ async def get_user_by_email(active_username: str, email: str, request_id: str) - """ Retrieve a user by email. """ - logger.info(f'{request_id} | Getting user by email: {email}') + logger.info(f'Getting user by email: {email}') user = await db_find_one(user_collection, {'email': email}) if not user: - logger.error(f'{request_id} | User retrieval failed with code USR002') + logger.error(f'User retrieval failed with code USR002') return ResponseModel( status_code=404, response_headers={'request_id': request_id}, @@ -136,11 +136,11 @@ async def get_user_by_email(active_username: str, email: str, request_id: str) - del user['_id'] if 'password' in user: del user['password'] - logger.info(f'{request_id} | User retrieval successful') + logger.info(f'User retrieval successful') if not active_username == user.get('username') and not await platform_role_required_bool( active_username, 'manage_users' ): - logger.error(f'{request_id} | User retrieval failed with code USR008') + logger.error(f'User retrieval failed with code USR008') return ResponseModel( status_code=403, error_code='USR008', @@ -153,11 +153,11 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: """ Create a new user. """ - logger.info(f'{request_id} | Creating user: {data.username}') + logger.info(f'Creating user: {data.username}') try: if data.custom_attributes is not None and len(data.custom_attributes.keys()) > 10: logger.error( - f'{request_id} | User creation failed with code USR016: Too many custom attributes' + f'User creation failed with code USR016: Too many custom attributes' ) return ResponseModel( status_code=400, @@ -167,7 +167,7 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: ).dict() except Exception: logger.error( - f'{request_id} | User creation failed with code USR016: Invalid custom attributes payload' + f'User creation failed with code USR016: Invalid custom attributes payload' ) return ResponseModel( status_code=400, @@ -176,7 +176,7 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: error_message='Maximum 10 custom attributes allowed. Please replace an existing one.', ).dict() if await db_find_one(user_collection, {'username': data.username}): - logger.error(f'{request_id} | User creation failed with code USR001') + logger.error(f'User creation failed with code USR001') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -184,7 +184,7 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: error_message='Username already exists', ).dict() if await db_find_one(user_collection, {'email': data.email}): - logger.error(f'{request_id} | User creation failed with code USR001') + logger.error(f'User creation failed with code USR001') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -192,7 +192,7 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: error_message='Email already exists', ).dict() if not password_util.is_secure_password(data.password): - logger.error(f'{request_id} | User creation failed with code USR005') + logger.error(f'User creation failed with code USR005') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -207,7 +207,7 @@ async def create_user(data: CreateUserModel, request_id: str) -> dict: if 'password' in data_dict: del data_dict['password'] doorman_cache.set_cache('user_cache', data.username, data_dict) - logger.info(f'{request_id} | User creation successful') + logger.info(f'User creation successful') return ResponseModel( status_code=201, response_headers={'request_id': request_id}, @@ -239,12 +239,12 @@ async def update_user(username: str, update_data: dict, request_id: str) -> dict """ Update user information. """ - logger.info(f'{request_id} | Updating user: {username}') + logger.info(f'Updating user: {username}') user = doorman_cache.get_cache('user_cache', username) if not user: user = await db_find_one(user_collection, {'username': username}) if not user: - logger.error(f'{request_id} | User update failed with code USR002') + logger.error(f'User update failed with code USR002') return ResponseModel( status_code=404, error_code='USR002', error_message='User not found' ).dict() @@ -258,7 +258,7 @@ async def update_user(username: str, update_data: dict, request_id: str) -> dict and len(non_null_update_data['custom_attributes'].keys()) > 10 ): logger.error( - f'{request_id} | User update failed with code USR016: Too many custom attributes' + f'User update failed with code USR016: Too many custom attributes' ) return ResponseModel( status_code=400, @@ -267,7 +267,7 @@ async def update_user(username: str, update_data: dict, request_id: str) -> dict ).dict() except Exception: logger.error( - f'{request_id} | User update failed with code USR016: Invalid custom attributes payload' + f'User update failed with code USR016: Invalid custom attributes payload' ) return ResponseModel( status_code=400, @@ -282,19 +282,19 @@ async def update_user(username: str, update_data: dict, request_id: str) -> dict if update_result.modified_count > 0: doorman_cache.delete_cache('user_cache', username) if not update_result.acknowledged or update_result.modified_count == 0: - logger.error(f'{request_id} | User update failed with code USR003') + logger.error(f'User update failed with code USR003') return ResponseModel( status_code=400, error_code='USR004', error_message='Unable to update user' ).dict() except Exception as e: doorman_cache.delete_cache('user_cache', username) logger.error( - f'{request_id} | User update failed with exception: {str(e)}', exc_info=True + f'User update failed with exception: {str(e)}', exc_info=True ) raise if non_null_update_data.get('role'): await UserService.purge_apis_after_role_change(username, request_id) - logger.info(f'{request_id} | User update successful') + logger.info(f'User update successful') return ResponseModel( status_code=200, response_headers={'request_id': request_id}, @@ -306,18 +306,18 @@ async def delete_user(username: str, request_id: str) -> dict: """ Delete a user. """ - logger.info(f'{request_id} | Deleting user: {username}') + logger.info(f'Deleting user: {username}') user = doorman_cache.get_cache('user_cache', username) if not user: user = await db_find_one(user_collection, {'username': username}) if not user: - logger.error(f'{request_id} | User deletion failed with code USR002') + logger.error(f'User deletion failed with code USR002') return ResponseModel( status_code=404, error_code='USR002', error_message='User not found' ).dict() delete_result = await db_delete_one(user_collection, {'username': username}) if not delete_result.acknowledged or delete_result.deleted_count == 0: - logger.error(f'{request_id} | User deletion failed with code USR003') + logger.error(f'User deletion failed with code USR003') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -326,7 +326,7 @@ async def delete_user(username: str, request_id: str) -> dict: ).dict() doorman_cache.delete_cache('user_cache', username) doorman_cache.delete_cache('user_subscription_cache', username) - logger.info(f'{request_id} | User deletion successful') + logger.info(f'User deletion successful') return ResponseModel( status_code=200, response_headers={'request_id': request_id}, @@ -338,9 +338,9 @@ async def update_password(username: str, update_data: dict, request_id: str) -> """ Update user information. """ - logger.info(f'{request_id} | Updating password for user: {username}') + logger.info(f'Updating password for user: {username}') if not password_util.is_secure_password(update_data.new_password): - logger.error(f'{request_id} | User password update failed with code USR005') + logger.error(f'User password update failed with code USR005') return ResponseModel( status_code=400, response_headers={'request_id': request_id}, @@ -357,13 +357,13 @@ async def update_password(username: str, update_data: dict, request_id: str) -> except Exception as e: doorman_cache.delete_cache('user_cache', username) logger.error( - f'{request_id} | User password update failed with exception: {str(e)}', + f'User password update failed with exception: {str(e)}', exc_info=True, ) raise user = await db_find_one(user_collection, {'username': username}) if not user: - logger.error(f'{request_id} | User password update failed with code USR002') + logger.error(f'User password update failed with code USR002') return ResponseModel( status_code=404, response_headers={'request_id': request_id}, @@ -375,7 +375,7 @@ async def update_password(username: str, update_data: dict, request_id: str) -> if 'password' in user: del user['password'] doorman_cache.set_cache('user_cache', username, user) - logger.info(f'{request_id} | User password update successful') + logger.info(f'User password update successful') return ResponseModel( status_code=200, response_headers={'request_id': request_id}, @@ -387,7 +387,7 @@ async def purge_apis_after_role_change(username: str, request_id: str) -> None: """ Remove subscriptions after role change. """ - logger.info(f'{request_id} | Purging APIs for user: {username}') + logger.info(f'Purging APIs for user: {username}') user_subscriptions = doorman_cache.get_cache( 'user_subscription_cache', username ) or await db_find_one(subscriptions_collection, {'username': username}) @@ -416,18 +416,18 @@ async def purge_apis_after_role_change(username: str, request_id: str) -> None: except Exception as e: doorman_cache.delete_cache('user_subscription_cache', username) logger.error( - f'{request_id} | Subscription update failed with exception: {str(e)}', + f'Subscription update failed with exception: {str(e)}', exc_info=True, ) raise - logger.info(f'{request_id} | Purge successful') + logger.info(f'Purge successful') @staticmethod async def get_all_users(page: int, page_size: int, request_id: str) -> dict: """ Get all users. """ - logger.info(f'{request_id} | Getting all users: Page={page} Page Size={page_size}') + logger.info(f'Getting all users: Page={page} Page Size={page_size}') try: page, page_size = validate_page_params(page, page_size) except Exception as e: @@ -467,7 +467,7 @@ async def get_all_users(page: int, page_size: int, request_id: str) -> dict: for key, value in user.items(): if isinstance(value, bytes): user[key] = value.decode('utf-8') - logger.info(f'{request_id} | User retrieval successful') + logger.info(f'User retrieval successful') return ResponseModel( status_code=200, response={ diff --git a/backend-services/services/vault_service.py b/backend-services/services/vault_service.py index cc814308..934dc008 100644 --- a/backend-services/services/vault_service.py +++ b/backend-services/services/vault_service.py @@ -42,12 +42,12 @@ async def create_vault_entry( ResponseModel dict with success or error """ logger.info( - f'{request_id} | Creating vault entry: {entry_data.key_name} for user: {username}' + f'Creating vault entry: {entry_data.key_name} for user: {username}' ) # Check if VAULT_KEY is configured if not is_vault_configured(): - logger.error(f'{request_id} | VAULT_KEY not configured') + logger.error(f'VAULT_KEY not configured') return ResponseModel( status_code=500, error_code='VAULT001', @@ -57,14 +57,14 @@ async def create_vault_entry( # Get user to retrieve email user = await db_find_one(user_collection, {'username': username}) if not user: - logger.error(f'{request_id} | User not found: {username}') + logger.error(f'User not found: {username}') return ResponseModel( status_code=404, error_code='VAULT002', error_message='User not found' ).dict() email = user.get('email') if not email: - logger.error(f'{request_id} | User email not found: {username}') + logger.error(f'User email not found: {username}') return ResponseModel( status_code=400, error_code='VAULT003', @@ -76,7 +76,7 @@ async def create_vault_entry( vault_entries_collection, {'username': username, 'key_name': entry_data.key_name} ) if existing: - logger.error(f'{request_id} | Vault entry already exists: {entry_data.key_name}') + logger.error(f'Vault entry already exists: {entry_data.key_name}') return ResponseModel( status_code=409, error_code='VAULT004', @@ -87,7 +87,7 @@ async def create_vault_entry( try: encrypted_value = encrypt_vault_value(entry_data.value, email, username) except Exception as e: - logger.error(f'{request_id} | Encryption failed: {str(e)}') + logger.error(f'Encryption failed: {str(e)}') return ResponseModel( status_code=500, error_code='VAULT005', @@ -109,7 +109,7 @@ async def create_vault_entry( result = await db_insert_one(vault_entries_collection, vault_entry) if result.acknowledged: logger.info( - f'{request_id} | Vault entry created successfully: {entry_data.key_name}' + f'Vault entry created successfully: {entry_data.key_name}' ) return ResponseModel( status_code=201, @@ -117,14 +117,14 @@ async def create_vault_entry( data={'key_name': entry_data.key_name}, ).dict() else: - logger.error(f'{request_id} | Failed to create vault entry') + logger.error(f'Failed to create vault entry') return ResponseModel( status_code=500, error_code='VAULT006', error_message='Failed to create vault entry', ).dict() except Exception as e: - logger.error(f'{request_id} | Error creating vault entry: {str(e)}') + logger.error(f'Error creating vault entry: {str(e)}') return ResponseModel( status_code=500, error_code=ErrorCodes.UNEXPECTED, error_message=Messages.UNEXPECTED ).dict() @@ -142,14 +142,14 @@ async def get_vault_entry(username: str, key_name: str, request_id: str) -> dict Returns: ResponseModel dict with vault entry (without value) or error """ - logger.info(f'{request_id} | Getting vault entry: {key_name} for user: {username}') + logger.info(f'Getting vault entry: {key_name} for user: {username}') entry = await db_find_one( vault_entries_collection, {'username': username, 'key_name': key_name} ) if not entry: - logger.error(f'{request_id} | Vault entry not found: {key_name}') + logger.error(f'Vault entry not found: {key_name}') return ResponseModel( status_code=404, error_code='VAULT007', error_message='Vault entry not found' ).dict() @@ -174,7 +174,7 @@ async def list_vault_entries(username: str, request_id: str) -> dict: Returns: ResponseModel dict with list of vault entries (without values) """ - logger.info(f'{request_id} | Listing vault entries for user: {username}') + logger.info(f'Listing vault entries for user: {username}') try: entries = await db_find_list(vault_entries_collection, {'username': username}) @@ -192,7 +192,7 @@ async def list_vault_entries(username: str, request_id: str) -> dict: status_code=200, data={'entries': clean_entries, 'count': len(clean_entries)} ).dict() except Exception as e: - logger.error(f'{request_id} | Error listing vault entries: {str(e)}') + logger.error(f'Error listing vault entries: {str(e)}') return ResponseModel( status_code=500, error_code=ErrorCodes.UNEXPECTED, error_message=Messages.UNEXPECTED ).dict() @@ -213,7 +213,7 @@ async def update_vault_entry( Returns: ResponseModel dict with success or error """ - logger.info(f'{request_id} | Updating vault entry: {key_name} for user: {username}') + logger.info(f'Updating vault entry: {key_name} for user: {username}') # Check if entry exists entry = await db_find_one( @@ -221,7 +221,7 @@ async def update_vault_entry( ) if not entry: - logger.error(f'{request_id} | Vault entry not found: {key_name}') + logger.error(f'Vault entry not found: {key_name}') return ResponseModel( status_code=404, error_code='VAULT007', error_message='Vault entry not found' ).dict() @@ -241,17 +241,17 @@ async def update_vault_entry( ) if result.modified_count > 0: - logger.info(f'{request_id} | Vault entry updated successfully: {key_name}') + logger.info(f'Vault entry updated successfully: {key_name}') return ResponseModel( status_code=200, message='Vault entry updated successfully' ).dict() else: - logger.warning(f'{request_id} | No changes made to vault entry: {key_name}') + logger.warning(f'No changes made to vault entry: {key_name}') return ResponseModel( status_code=200, message='No changes made to vault entry' ).dict() except Exception as e: - logger.error(f'{request_id} | Error updating vault entry: {str(e)}') + logger.error(f'Error updating vault entry: {str(e)}') return ResponseModel( status_code=500, error_code=ErrorCodes.UNEXPECTED, error_message=Messages.UNEXPECTED ).dict() @@ -269,7 +269,7 @@ async def delete_vault_entry(username: str, key_name: str, request_id: str) -> d Returns: ResponseModel dict with success or error """ - logger.info(f'{request_id} | Deleting vault entry: {key_name} for user: {username}') + logger.info(f'Deleting vault entry: {key_name} for user: {username}') # Check if entry exists entry = await db_find_one( @@ -277,7 +277,7 @@ async def delete_vault_entry(username: str, key_name: str, request_id: str) -> d ) if not entry: - logger.error(f'{request_id} | Vault entry not found: {key_name}') + logger.error(f'Vault entry not found: {key_name}') return ResponseModel( status_code=404, error_code='VAULT007', error_message='Vault entry not found' ).dict() @@ -288,19 +288,19 @@ async def delete_vault_entry(username: str, key_name: str, request_id: str) -> d ) if result.deleted_count > 0: - logger.info(f'{request_id} | Vault entry deleted successfully: {key_name}') + logger.info(f'Vault entry deleted successfully: {key_name}') return ResponseModel( status_code=200, message='Vault entry deleted successfully' ).dict() else: - logger.error(f'{request_id} | Failed to delete vault entry: {key_name}') + logger.error(f'Failed to delete vault entry: {key_name}') return ResponseModel( status_code=500, error_code='VAULT008', error_message='Failed to delete vault entry', ).dict() except Exception as e: - logger.error(f'{request_id} | Error deleting vault entry: {str(e)}') + logger.error(f'Error deleting vault entry: {str(e)}') return ResponseModel( status_code=500, error_code=ErrorCodes.UNEXPECTED, error_message=Messages.UNEXPECTED ).dict() diff --git a/backend-services/tests/test_global_logging.py b/backend-services/tests/test_global_logging.py new file mode 100644 index 00000000..bfdf02f4 --- /dev/null +++ b/backend-services/tests/test_global_logging.py @@ -0,0 +1,36 @@ +import logging +import pytest +from httpx import AsyncClient + +@pytest.mark.asyncio +async def test_global_logging_middleware_404(): + from doorman import doorman + import logging + + # Custom handler to capture logs + class ListHandler(logging.Handler): + def __init__(self): + super().__init__() + self.records = [] + def emit(self, record): + self.records.append(self.format(record)) + + logger = logging.getLogger('doorman.gateway') + handler = ListHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + try: + async with AsyncClient(app=doorman, base_url='http://testserver') as ac: + path = '/some/random/path/that/does/not/exist' + r = await ac.get(path) + assert r.status_code == 404 + + # Verify log entry exists in captured records + log_text = '\n'.join(handler.records) + assert 'status_code: 404' in log_text + assert path in log_text + finally: + logger.removeHandler(handler) + diff --git a/backend-services/tests/test_logging_redaction.py b/backend-services/tests/test_logging_redaction.py index f68741df..368cd804 100644 --- a/backend-services/tests/test_logging_redaction.py +++ b/backend-services/tests/test_logging_redaction.py @@ -5,8 +5,11 @@ def test_logging_redaction_filters_sensitive_values(): logger = logging.getLogger('doorman.gateway') filt = None for h in logger.handlers: - if h.filters: - filt = h.filters[0] + for f in h.filters: + if type(f).__name__ == 'RedactFilter': + filt = f + break + if filt: break assert filt is not None, 'Redaction filter not configured' diff --git a/backend-services/tests/test_logging_redaction_extended.py b/backend-services/tests/test_logging_redaction_extended.py index c0760178..15aaa496 100644 --- a/backend-services/tests/test_logging_redaction_extended.py +++ b/backend-services/tests/test_logging_redaction_extended.py @@ -5,8 +5,11 @@ def test_redaction_handles_cookies_csrf_and_mixed_cases(): logger = logging.getLogger('doorman.gateway') filt = None for h in logger.handlers: - if h.filters: - filt = h.filters[0] + for f in h.filters: + if type(f).__name__ == 'RedactFilter': + filt = f + break + if filt: break assert filt is not None diff --git a/backend-services/tests/test_self_service_privilege_escalation.py b/backend-services/tests/test_self_service_privilege_escalation.py new file mode 100644 index 00000000..ff500280 --- /dev/null +++ b/backend-services/tests/test_self_service_privilege_escalation.py @@ -0,0 +1,253 @@ +""" +Tests for self-service profile update privilege escalation prevention. + +CVE fix: Users without manage_users permission should not be able to modify +restricted fields (role, groups, active, username) when updating their own profile. + +These are integration tests that use the actual application fixtures. +""" + +import pytest + + +@pytest.mark.asyncio +async def test_regular_user_cannot_change_own_role(authed_client): + """User cannot escalate privileges by changing their own role.""" + # Create a role without manage_users + await authed_client.post( + '/platform/role', + json={'role_name': 'limited', 'role_description': 'Limited user', 'manage_users': False} + ) + + # Create user with the limited role + await authed_client.post( + '/platform/user', + json={ + 'username': 'privesc_test_user', + 'email': 'privesc@doorman.dev', + 'password': 'SecurePassword123!', + 'role': 'limited', + 'active': True, + } + ) + + # Login as that limited user + from doorman import doorman + from httpx import AsyncClient + import os + + user_client = AsyncClient(app=doorman, base_url='http://testserver') + r = await user_client.post( + '/platform/authorization', + json={'email': 'privesc@doorman.dev', 'password': 'SecurePassword123!'} + ) + assert r.status_code == 200 + body = r.json() + token = body.get('access_token') + if token: + user_client.cookies.set( + 'access_token_cookie', + token, + domain=os.environ.get('COOKIE_DOMAIN') or 'testserver', + path='/', + ) + + # Attempt privilege escalation: change own role to admin + resp = await user_client.put( + '/platform/user/privesc_test_user', + json={'role': 'admin'} + ) + + assert resp.status_code == 403, f'Expected 403, got {resp.status_code}: {resp.text}' + data = resp.json() + assert data.get('error_code') == 'USR023' + assert 'role' in data.get('error_message', '').lower() + + +@pytest.mark.asyncio +async def test_regular_user_cannot_change_own_groups(authed_client): + """User cannot escalate privileges by adding themselves to privileged groups.""" + # Create a role without manage_users + await authed_client.post( + '/platform/role', + json={'role_name': 'limited2', 'role_description': 'Limited user', 'manage_users': False} + ) + + # Create user with the limited role + await authed_client.post( + '/platform/user', + json={ + 'username': 'privesc_test_groups', + 'email': 'privesc_groups@doorman.dev', + 'password': 'SecurePassword123!', + 'role': 'limited2', + 'active': True, + } + ) + + # Login as that limited user + from doorman import doorman + from httpx import AsyncClient + import os + + user_client = AsyncClient(app=doorman, base_url='http://testserver') + r = await user_client.post( + '/platform/authorization', + json={'email': 'privesc_groups@doorman.dev', 'password': 'SecurePassword123!'} + ) + assert r.status_code == 200 + body = r.json() + token = body.get('access_token') + if token: + user_client.cookies.set( + 'access_token_cookie', + token, + domain=os.environ.get('COOKIE_DOMAIN') or 'testserver', + path='/', + ) + + # Attempt privilege escalation: add self to all groups + resp = await user_client.put( + '/platform/user/privesc_test_groups', + json={'groups': ['ALL', 'admin-group']} + ) + + assert resp.status_code == 403, f'Expected 403, got {resp.status_code}: {resp.text}' + data = resp.json() + assert data.get('error_code') == 'USR023' + assert 'groups' in data.get('error_message', '').lower() + + +@pytest.mark.asyncio +async def test_regular_user_cannot_change_own_active_status(authed_client): + """User cannot modify their own active status without manage_users.""" + # Create a role without manage_users + await authed_client.post( + '/platform/role', + json={'role_name': 'limited3', 'role_description': 'Limited user', 'manage_users': False} + ) + + # Create user with the limited role + await authed_client.post( + '/platform/user', + json={ + 'username': 'privesc_test_active', + 'email': 'privesc_active@doorman.dev', + 'password': 'SecurePassword123!', + 'role': 'limited3', + 'active': True, + } + ) + + # Login as that limited user + from doorman import doorman + from httpx import AsyncClient + import os + + user_client = AsyncClient(app=doorman, base_url='http://testserver') + r = await user_client.post( + '/platform/authorization', + json={'email': 'privesc_active@doorman.dev', 'password': 'SecurePassword123!'} + ) + assert r.status_code == 200 + body = r.json() + token = body.get('access_token') + if token: + user_client.cookies.set( + 'access_token_cookie', + token, + domain=os.environ.get('COOKIE_DOMAIN') or 'testserver', + path='/', + ) + + # Attempt privilege escalation: change own active flag + resp = await user_client.put( + '/platform/user/privesc_test_active', + json={'active': False} + ) + + assert resp.status_code == 403, f'Expected 403, got {resp.status_code}: {resp.text}' + data = resp.json() + assert data.get('error_code') == 'USR023' + assert 'active' in data.get('error_message', '').lower() + + +@pytest.mark.asyncio +async def test_regular_user_can_change_own_email(authed_client): + """User CAN change their own email without manage_users (allowed field).""" + # Create a role without manage_users + await authed_client.post( + '/platform/role', + json={'role_name': 'limited4', 'role_description': 'Limited user', 'manage_users': False} + ) + + # Create user with the limited role + await authed_client.post( + '/platform/user', + json={ + 'username': 'privesc_test_email', + 'email': 'privesc_email@doorman.dev', + 'password': 'SecurePassword123!', + 'role': 'limited4', + 'active': True, + } + ) + + # Login as that limited user + from doorman import doorman + from httpx import AsyncClient + import os + + user_client = AsyncClient(app=doorman, base_url='http://testserver') + r = await user_client.post( + '/platform/authorization', + json={'email': 'privesc_email@doorman.dev', 'password': 'SecurePassword123!'} + ) + assert r.status_code == 200 + body = r.json() + token = body.get('access_token') + if token: + user_client.cookies.set( + 'access_token_cookie', + token, + domain=os.environ.get('COOKIE_DOMAIN') or 'testserver', + path='/', + ) + + # Change own email - should be ALLOWED + resp = await user_client.put( + '/platform/user/privesc_test_email', + json={'email': 'new_email@doorman.dev'} + ) + + assert resp.status_code == 200, f'Expected 200, got {resp.status_code}: {resp.text}' + + +@pytest.mark.asyncio +async def test_admin_can_change_any_user_role(authed_client): + """Admin with manage_users CAN change another user's role.""" + # Create a limited role + await authed_client.post( + '/platform/role', + json={'role_name': 'target_role', 'role_description': 'Target user', 'manage_users': False} + ) + + # Create a target user + await authed_client.post( + '/platform/user', + json={ + 'username': 'target_user', + 'email': 'target@doorman.dev', + 'password': 'SecurePassword123!', + 'role': 'target_role', + 'active': True, + } + ) + + # Admin (authed_client) changes target user's role - should be ALLOWED + resp = await authed_client.put( + '/platform/user/target_user', + json={'role': 'admin'} + ) + + assert resp.status_code == 200, f'Expected 200, got {resp.status_code}: {resp.text}' diff --git a/backend-services/utils/analytics_aggregator.py b/backend-services/utils/analytics_aggregator.py index 7d318830..25ec24ad 100644 --- a/backend-services/utils/analytics_aggregator.py +++ b/backend-services/utils/analytics_aggregator.py @@ -186,6 +186,7 @@ def _aggregate_buckets( status_counts=dict(status_counts), api_counts=dict(api_counts), percentiles=percentiles, + unique_users_set=unique_users, ) def _aggregate_aggregated_buckets( @@ -214,8 +215,17 @@ def _aggregate_aggregated_buckets( # Ideally, we'd re-calculate from raw latencies, but those aren't stored in aggregated buckets weighted_percentiles = self._weighted_average_percentiles(buckets) - # Unique users: sum (may overcount, but acceptable for aggregated data) - unique_users = sum(b.unique_users for b in buckets) + # Unique users: merge sets if available, otherwise fallback to sum (imperfect but best effort) + unique_users_set = set() + has_sets = any(bool(b.unique_users_set) for b in buckets) + + if has_sets: + for b in buckets: + if b.unique_users_set: + unique_users_set.update(b.unique_users_set) + unique_users_count = len(unique_users_set) + else: + unique_users_count = sum(b.unique_users for b in buckets) return AggregatedMetrics( start_ts=start_ts, @@ -226,10 +236,11 @@ def _aggregate_aggregated_buckets( total_ms=total_ms, bytes_in=total_bytes_in, bytes_out=total_bytes_out, - unique_users=unique_users, + unique_users=unique_users_count, status_counts=dict(status_counts), api_counts=dict(api_counts), percentiles=weighted_percentiles, + unique_users_set=unique_users_set, ) def _weighted_average_percentiles( @@ -320,11 +331,31 @@ def to_dict(self) -> dict: def load_dict(self, data: dict) -> None: """Load aggregator state from persistence.""" - # Note: This is a simplified version. Full implementation would - # reconstruct AggregatedMetrics objects from dictionaries self._last_5min_aggregation = data.get('last_5min_aggregation', 0) self._last_hourly_aggregation = data.get('last_hourly_aggregation', 0) self._last_daily_aggregation = data.get('last_daily_aggregation', 0) + + # Restore buckets + self.five_minute_buckets.clear() + for b_data in data.get('five_minute_buckets', []): + try: + self.five_minute_buckets.append(AggregatedMetrics.from_dict(b_data)) + except Exception: + continue + + self.hourly_buckets.clear() + for b_data in data.get('hourly_buckets', []): + try: + self.hourly_buckets.append(AggregatedMetrics.from_dict(b_data)) + except Exception: + continue + + self.daily_buckets.clear() + for b_data in data.get('daily_buckets', []): + try: + self.daily_buckets.append(AggregatedMetrics.from_dict(b_data)) + except Exception: + continue # Global aggregator instance diff --git a/backend-services/utils/api_util.py b/backend-services/utils/api_util.py index 47f79099..30ece4b9 100644 --- a/backend-services/utils/api_util.py +++ b/backend-services/utils/api_util.py @@ -45,32 +45,43 @@ async def get_api_endpoints(api_id: str) -> list | None: endpoints_list = await db_find_list(endpoint_collection, {'api_id': api_id}) if not endpoints_list: return None - endpoints = [ - f'{endpoint.get("endpoint_method")}{endpoint.get("endpoint_uri")}' - for endpoint in endpoints_list - ] + endpoints = [] + for endpoint in endpoints_list: + # Use client_uri if available for routing matching + uri = endpoint.get('client_uri') or endpoint.get('endpoint_uri') + endpoints.append(f'{endpoint.get("endpoint_method")}{uri}') doorman_cache.set_cache('api_endpoint_cache', api_id, endpoints) return endpoints -async def get_endpoint(api: dict, method: str, endpoint_uri: str) -> dict | None: - """Return the endpoint document for a given API, method, and uri. +async def get_endpoint(api: dict, method: str, routing_uri: str) -> dict | None: + """Return the endpoint document for a given API, method, and URI. + + Args: + api: API document + method: HTTP method + routing_uri: The URI requested by the client (matches client_uri or endpoint_uri) Uses the same cache key pattern as EndpointService to avoid duplicate queries. """ api_name = api.get('api_name') api_version = api.get('api_version') - cache_key = f'/{method}/{api_name}/{api_version}/{endpoint_uri}'.replace('//', '/') + cache_key = f'/{method}/{api_name}/{api_version}/{routing_uri}'.replace('//', '/') endpoint = doorman_cache.get_cache('endpoint_cache', cache_key) if endpoint: return endpoint + + # Search for either client_uri or endpoint_uri matching the request doc = await db_find_one( endpoint_collection, { 'api_name': api_name, 'api_version': api_version, - 'endpoint_uri': endpoint_uri, 'endpoint_method': method, + '$or': [ + {'client_uri': routing_uri}, + {'endpoint_uri': routing_uri} + ] }, ) if not doc: diff --git a/backend-services/utils/correlation_util.py b/backend-services/utils/correlation_util.py index 3c56c271..8b1b50c3 100644 --- a/backend-services/utils/correlation_util.py +++ b/backend-services/utils/correlation_util.py @@ -40,12 +40,11 @@ def ensure_correlation_id() -> str: def log_with_correlation(level: str, message: str, **kwargs) -> None: """ - Log a message with the correlation ID automatically prepended. + Log a message. The RequestIdFilter will automatically attach the ID. """ - cid = get_correlation_id() or 'no-correlation-id' - log_message = f'{cid} | {message}' + # cid = get_correlation_id() or 'no-correlation-id' # handled by filter log_func = getattr(logger, level.lower(), logger.info) - log_func(log_message, **kwargs) + log_func(message, **kwargs) async def run_with_correlation(coro, correlation_id_value: str | None = None): @@ -93,3 +92,19 @@ async def run_async_with_correlation( except Exception as e: log_with_correlation('error', f'Async task failed: {str(e)}', exc_info=True) raise + + +class RequestIdFilter(logging.Filter): + """ + Log filter that injects the current request/correlation ID into the log record. + This ensures that all logs, even those without explicit IDs, are tagged. + """ + + def filter(self, record): + cid = get_correlation_id() + # Always set request_id to avoid formatting errors + record.request_id = cid if cid else 'no-request-id' + + # We generally rely on the formatter to include the request_id. + # Prepending it here would cause duplication (e.g. "ID | ID | message"). + return True diff --git a/backend-services/utils/enhanced_metrics_util.py b/backend-services/utils/enhanced_metrics_util.py index c3a42778..42f86567 100644 --- a/backend-services/utils/enhanced_metrics_util.py +++ b/backend-services/utils/enhanced_metrics_util.py @@ -267,7 +267,25 @@ def get_snapshot( # Count unique users unique_users = set() for bucket in buckets: - unique_users.update(bucket.unique_users) + # EnhancedMinuteBucket has unique_users as set + if hasattr(bucket, 'unique_users') and isinstance(bucket.unique_users, set): + unique_users.update(bucket.unique_users) + # AggregatedMetrics has unique_users_set + elif hasattr(bucket, 'unique_users_set') and isinstance(bucket.unique_users_set, set): + unique_users.update(bucket.unique_users_set) + # MinuteBucket has user_counts keys + elif hasattr(bucket, 'user_counts') and bucket.user_counts: + unique_users.update(bucket.user_counts.keys()) + + unique_users_count = len(unique_users) + if unique_users_count == 0 and buckets: + # Fallback: sum counts if we have no actual user identifiers + unique_users_count = sum( + getattr(b, 'unique_users', 0) + if not isinstance(getattr(b, 'unique_users', 0), (set, list)) + else len(getattr(b, 'unique_users', [])) + for b in buckets + ) # Aggregate status counts status_distribution: dict[str, int] = defaultdict(int) @@ -373,7 +391,7 @@ def get_snapshot( percentiles=percentiles, total_bytes_in=total_bytes_in, total_bytes_out=total_bytes_out, - unique_users=len(unique_users), + unique_users=unique_users_count, series=series, top_apis=sorted(api_counts.items(), key=lambda x: x[1], reverse=True)[:10], top_users=sorted(user_counts.items(), key=lambda x: x[1], reverse=True)[:10], @@ -387,7 +405,24 @@ def snapshot(self, range_key: str, group: str = 'minute', sort: str = 'asc') -> """ range_to_minutes = {'1h': 60, '24h': 60 * 24, '7d': 60 * 24 * 7, '30d': 60 * 24 * 30} minutes = range_to_minutes.get(range_key, 60 * 24) - buckets: list[EnhancedMinuteBucket] = list(self._buckets)[-minutes:] + + # Use simple deque slice if within memory range + if minutes <= self._max_minutes: + buckets: list[EnhancedMinuteBucket] = list(self._buckets)[-minutes:] + else: + # Require historical data from aggregator + end_ts = int(time.time()) + start_ts = end_ts - (minutes * 60) + + # Fetch aggregated buckets + # Note: returns AggregatedMetrics, not EnhancedMinuteBucket + # but they share key attributes (count, error_count, total_ms, etc.) + buckets = analytics_aggregator.get_buckets_for_range(start_ts, end_ts) + + # If nothing returned, fallback to in-memory (better than nothing) + if not buckets: + buckets = list(self._buckets) + series = [] if group == 'day': @@ -403,7 +438,10 @@ def snapshot(self, range_key: str, group: str = 'minute', sort: str = 'asc') -> } ) for b in buckets: - day_ts = int((b.start_ts // 86400) * 86400) + # Handle both EnhancedMinuteBucket and AggregatedMetrics + # AggregatedMetrics might be daily or hourly, but we just group by day timestamp here + ts = getattr(b, 'start_ts') + day_ts = int((ts // 86400) * 86400) d = day_map[day_ts] d['count'] += b.count d['error_count'] += b.error_count @@ -453,7 +491,33 @@ def snapshot(self, range_key: str, group: str = 'minute', sort: str = 'asc') -> total = self.total_requests avg_total_ms = (self.total_ms / total) if total else 0.0 status = {str(k): v for k, v in self.status_counts.items()} + + unique_users = set() + for b in buckets: + # EnhancedMinuteBucket has unique_users as set + if hasattr(b, 'unique_users') and isinstance(b.unique_users, set): + unique_users.update(b.unique_users) + + # AggregatedMetrics has unique_users_set + elif hasattr(b, 'unique_users_set') and isinstance(b.unique_users_set, set): + unique_users.update(b.unique_users_set) + + # Fallback to user_counts keys (MinuteBucket compatibility) + elif getattr(b, 'user_counts', None): + unique_users.update(b.user_counts.keys()) + + unique_users_count = len(unique_users) + if unique_users_count == 0 and buckets: + # Fallback: sum counts if sets are not available + unique_users_count = sum( + getattr(b, 'unique_users', 0) + if not isinstance(getattr(b, 'unique_users', 0), (set, list)) + else len(getattr(b, 'unique_users', [])) + for b in buckets + ) + return { + 'unique_users': unique_users_count, 'total_requests': total, 'avg_response_ms': avg_total_ms, 'total_bytes_in': self.total_bytes_in, @@ -466,6 +530,7 @@ def snapshot(self, range_key: str, group: str = 'minute', sort: str = 'asc') -> :10 ], 'top_apis': sorted(self.api_counts.items(), key=lambda kv: kv[1], reverse=True)[:10], + 'buckets': [b.to_dict() for b in list(self._buckets)], } def save_to_file(self, path: str) -> None: @@ -484,6 +549,9 @@ def save_to_file(self, path: str) -> None: 'total_ms': self.total_ms, 'total_bytes_in': self.total_bytes_in, 'total_bytes_out': self.total_bytes_out, + 'status_counts': dict(self.status_counts), + 'username_counts': dict(self.username_counts), + 'api_counts': dict(self.api_counts), 'buckets': [b.to_dict() for b in list(self._buckets)], } with open(tmp, 'w', encoding='utf-8') as f: @@ -492,11 +560,41 @@ def save_to_file(self, path: str) -> None: except Exception: pass + def load_dict(self, data: dict) -> None: + """Populate store from dictionary.""" + try: + self.total_requests = int(data.get('total_requests', 0)) + self.total_ms = float(data.get('total_ms', 0.0)) + self.total_bytes_in = int(data.get('total_bytes_in', 0)) + self.total_bytes_out = int(data.get('total_bytes_out', 0)) + + self.status_counts = defaultdict(int, data.get('status_counts') or {}) + self.username_counts = defaultdict(int, data.get('username_counts') or {}) + self.api_counts = defaultdict(int, data.get('api_counts') or {}) + + self._buckets.clear() + for bd in data.get('buckets', []): + try: + self._buckets.append(EnhancedMinuteBucket.from_dict(bd)) + except Exception: + continue + except Exception: + pass + def load_from_file(self, path: str) -> None: """Load metrics from file.""" - # Note: Simplified version - full implementation would reconstruct - # EnhancedMinuteBucket objects from saved data - pass + try: + if not os.path.exists(path): + return + + import json + with open(path, encoding='utf-8') as f: + data = json.load(f) + + if isinstance(data, dict): + self.load_dict(data) + except Exception: + pass # Global enhanced metrics store instance diff --git a/backend-services/utils/limit_throttle_util.py b/backend-services/utils/limit_throttle_util.py index d77ec0c9..942fa84b 100644 --- a/backend-services/utils/limit_throttle_util.py +++ b/backend-services/utils/limit_throttle_util.py @@ -40,23 +40,26 @@ class InMemoryWindowCounter: def __init__(self): self._store = {} + self._lock = asyncio.Lock() async def incr(self, key: str) -> int: - now = int(time.time()) - entry = self._store.get(key) - if entry and entry['expires_at'] > now: - entry['count'] += 1 - else: - entry = {'count': 1, 'expires_at': now + 1} - self._store[key] = entry - return entry['count'] + async with self._lock: + now = int(time.time()) + entry = self._store.get(key) + if entry and entry['expires_at'] > now: + entry['count'] += 1 + else: + entry = {'count': 1, 'expires_at': now + 1} + self._store[key] = entry + return entry['count'] async def expire(self, key: str, ttl_seconds: int) -> None: - now = int(time.time()) - entry = self._store.get(key) - if entry: - entry['expires_at'] = now + int(ttl_seconds) - self._store[key] = entry + async with self._lock: + now = int(time.time()) + entry = self._store.get(key) + if entry: + entry['expires_at'] = now + int(ttl_seconds) + self._store[key] = entry _fallback_counter = InMemoryWindowCounter() diff --git a/fix.md b/fix.md deleted file mode 100644 index 1cab6a08..00000000 --- a/fix.md +++ /dev/null @@ -1,36 +0,0 @@ -# v1 Launch Fixes - -This file tracks potential issues, impact, and suggested fixes discovered during the pre-launch review. - -## Auth/session + UI access -- Cookie SameSite handling: `COOKIE_SAMESITE=None` on HTTP drops cookies in browsers. Fix: in `backend-services/routes/authorization_routes.py`, if `samesite == 'none'` and `_secure` is false, downgrade to `lax` and log a warning. This prevents confusing login loops in dev. - - - - -- Admin UI access inconsistency: login allows “super admin” regardless of `ui_access`, but `AuthContext` later sets `hasUIAccess` strictly from `user.ui_access`. Fix: align `AuthContext` to treat `role === 'admin'` or `username === 'admin'` as UI‑allowed (or ensure seed/restore always forces `ui_access=true`). - - - - -## Admin/user/role management -- `GET /platform/user/email/{email}` can 500 when user not found: `UserService.get_user_by_email` dereferences `user` before checking `None`. Fix: check `if not user` before accessing keys. -- Admin subscription bypass is broken: `subscription_required` compares `username` to `DOORMAN_ADMIN_EMAIL`, so admin users don’t bypass subscriptions unless username equals email. Fix: use `is_admin_user(username)` or `username == 'admin'` (or check role). -- Response schema mismatch: `GET /platform/user/email/{email}` declares `response_model=list[UserModelResponse]` but returns a single user object. Fix: adjust response_model or return list consistently. -- Super admin cannot see self: `/platform/user/all`, `/platform/user/{username}`, and `/platform/user/email/{email}` always hide `admin`. Fix: allow `admin` to see self when the requester is admin (or when `username == auth_username`). - -## Gateway routing + subscriptions -- Subscriptions allow horizontal privilege escalation: `/platform/subscription/subscribe` and `/platform/subscription/unsubscribe` allow any authenticated user to target any username; only group membership is checked. Fix: if `api_data.username != actor`, require `manage_subscriptions` (or admin) before proceeding. -- Subscription reads lack permission checks: `/platform/subscription/subscriptions/{user_id}` returns any user’s subscriptions to any authenticated caller. Fix: restrict to self or require `manage_subscriptions`. -- API allowed roles are defined but never enforced on gateway requests. If roles are intended to gate API access, add a role check alongside group/subscription enforcement; otherwise clarify docs/UI to avoid false expectations. - -## Analytics/logs/dashboard + config/security -- Config export APIs use `cursor.to_list()` on sync PyMongo cursors (`backend-services/routes/config_routes.py`). In Mongo mode this raises `AttributeError`. Fix: replace with `list(cursor)` or use async db helpers consistently. -- Subscriptions available-apis uses `cursor.to_list()` on sync cursor (`backend-services/routes/subscription_routes.py`); same issue in Mongo mode. Fix: convert to list like above or use async helper. -- Dashboard data requires only auth, not `view_analytics`. If analytics visibility is supposed to be permissioned, add a role check (or clarify that dashboard is always visible to UI‑access users). - -## Tests/smoke coverage -- No coverage for subscription permission checks (subscribe/unsubscribe another user, read others’ subscriptions). Add tests to prevent privilege escalation regressions. -- No coverage for API role enforcement on gateway access (if intended). Add a test that denies a user whose role is not in `api_allowed_roles`. -- No coverage for cookie SameSite/Secure behavior in HTTP dev. Add a web client smoke test or integration test that validates cookie presence after login when `HTTPS_ONLY=false`. -- Config export/import is only exercised in memory mode; no test for Mongo mode cursor handling. Add a small integration test or adjust code to use driver‑agnostic helper. diff --git a/web-client/src/app/apis/[apiId]/endpoints/add/page.tsx b/web-client/src/app/apis/[apiId]/endpoints/add/page.tsx index cf997f7c..f0fb3c51 100644 --- a/web-client/src/app/apis/[apiId]/endpoints/add/page.tsx +++ b/web-client/src/app/apis/[apiId]/endpoints/add/page.tsx @@ -21,6 +21,7 @@ export default function AddEndpointPage() { const [method, setMethod] = useState('GET') const [uri, setUri] = useState('') + const [clientUri, setClientUri] = useState('') const [description, setDescription] = useState('') const [useOverride, setUseOverride] = useState(false) const [servers, setServers] = useState([]) @@ -34,7 +35,7 @@ export default function AddEndpointPage() { setApiName(parsed.api_name || '') setApiVersion(parsed.api_version || '') } - } catch {} + } catch { } }, []) const addServer = () => { @@ -62,6 +63,9 @@ export default function AddEndpointPage() { endpoint_uri: uri.startsWith('/') ? uri : '/' + uri, endpoint_description: description || `${method} ${uri}` } + if (clientUri.trim()) { + body.client_uri = clientUri.trim() + } if (useOverride && servers.length > 0) { body.endpoint_servers = servers } @@ -69,7 +73,7 @@ export default function AddEndpointPage() { setSuccess('Endpoint created') setTimeout(() => setSuccess(null), 1500) router.push(`/apis/${encodeURIComponent(apiId)}/endpoints`) - } catch (e:any) { + } catch (e: any) { setError(e?.message || 'Failed to create endpoint') } finally { setLoading(false) @@ -108,7 +112,7 @@ export default function AddEndpointPage() {
@@ -116,7 +120,14 @@ export default function AddEndpointPage() { URI - setUri(e.target.value)} placeholder="/path/{id}" /> + setUri(e.target.value)} placeholder="/backend/path/{id}" /> +
+
+ + setClientUri(e.target.value)} placeholder="/public/path/{id}" />
@@ -125,7 +136,7 @@ export default function AddEndpointPage() {
- setUseOverride(e.target.checked)} /> + setUseOverride(e.target.checked)} />