diff --git a/.gitignore b/.gitignore index 5d6955d..d5bf056 100644 --- a/.gitignore +++ b/.gitignore @@ -535,3 +535,16 @@ volumes/configs/**/*.DS_Store volumes/configs/**/*Thumbs.db volumes/configs/**/*~ volumes/configs/**/*._* + +# Internal design docs (kept local, not published) +docs/superpowers/ + +# Internal benchmark/test bot fixtures (reveal scoring expectations — keep local) +benchmarks/ +tests/fixtures/bots/ + +# Private detector source (compiled to the rt_bv_score wheel — never publish source) +private/ +# Rust build artifacts +target/ +Cargo.lock diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..da79764 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/modules/rest.mdm-sn-container-runner"] + path = src/modules/rest.mdm-sn-container-runner + url = git@github.com:RedTeamSubnet/rest.hb-bot-executer.git diff --git a/compose.yml b/compose.yml index 4d997da..ea1beaf 100644 --- a/compose.yml +++ b/compose.yml @@ -3,11 +3,23 @@ services: image: redteamsubnet61/rest-bv-challenge:0.0.0 build: context: ./src/bv_challenge/challenge + platforms: + - linux/amd64 restart: unless-stopped + networks: + - bot-virus-challenge-net + - bot-executor-net + platform: linux/amd64 + privileged: true + ulimits: + nofile: 32768 environment: TERM: ${TERM:-xterm} TZ: ${TZ:-Asia/Seoul} BV_CHALLENGE_API_PORT: ${BV_CHALLENGE_API_PORT:-10001} + BV_CHALLENGE_API_BOT_RUNNER__URL: ${BV_CHALLENGE_API_BOT_RUNNER__URL:-http://bot-runner:8000} + BV_CHALLENGE_API_BOT_RUNNER__SESSION_COUNT: ${BV_CHALLENGE_API_BOT_RUNNER__SESSION_COUNT:-2} + BV_CHALLENGE_API_BOT_RUNNER__REQUEST_TIMEOUT_SEC: ${BV_CHALLENGE_API_BOT_RUNNER__REQUEST_TIMEOUT_SEC:-900} env_file: - path: .env required: false @@ -18,3 +30,50 @@ services: ports: - "${BV_CHALLENGE_API_PORT:-10001}:${BV_CHALLENGE_API_PORT:-10001}" tty: true + + bot-runner: + image: redteamsubnet61/bot_virus_bot_runner:latest + build: + context: ./src/modules/rest.mdm-sn-container-runner + platforms: + - linux/amd64 + restart: unless-stopped + networks: + - bot-virus-challenge-net + - bot-executor-net + privileged: true + user: "0:0" + environment: + TERM: ${TERM:-xterm} + TZ: ${TZ:-UTC} + VM_RUNNER_API_PORT: ${VM_RUNNER_API_PORT:-8000} + # Hostname the spawned bot container uses to reach the challenge API on + # bot-executor-net. Must match the challenge-api service name/alias on + # that network (default "challenger-api" does not resolve here). + CHALLENGE_VM_HOST: ${CHALLENGE_VM_HOST:-challenge-api} + CHALLENGE_VM_PORT: ${CHALLENGE_VM_PORT:-10001} + env_file: + - path: .env + required: false + volumes: + - "bot-runner-logs:${VM_RUNNER_API_LOGS_DIR:-/var/log/rest.vm-runner}" + - "bot-runner-data:${VM_RUNNER_API_DATA_DIR:-/var/lib/rest.vm-runner}" + - "/var/run/docker.sock:/var/run/docker.sock" + ports: + - "${VM_RUNNER_API_PORT:-8000}:${VM_RUNNER_API_PORT:-8000}" + tty: true + +networks: + bot-virus-challenge-net: + name: bot-virus-challenge-net + driver: bridge + driver_opts: + com.docker.network.driver.mtu: 1400 + bot-executor-net: + name: bot-executor-net + driver: bridge + internal: true + +volumes: + bot-runner-logs: + bot-runner-data: diff --git a/pyproject.toml b/pyproject.toml index 2d25dc6..8dcdea4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,14 +32,16 @@ classifiers = [ ] dynamic = ["version", "dependencies", "optional-dependencies"] -# [tool.setuptools.packages.find] -# where = ["src"] -# include = ["bv_challenge*"] -# namespaces = false +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] +include = ["bv_challenge*"] [tool.setuptools.dynamic] -version = { attr = "bv_challenge.__version__.__version__" } -dependencies = { file = "./requirements.txt" } +version = { attr = "bv_challenge.__version__" } +dependencies = { file = ["requirements.txt"] } [tool.setuptools.dynamic.optional-dependencies] # Options dependencies for DEVELOPMENT @@ -62,8 +64,8 @@ dev = { file = [ # venv = ".venv" [project.urls] -Homepage = "https://github.com/RedTeamSubnet/challenge-template" +Homepage = "https://github.com/RedTeamSubnet/bot-virus-challenge" Documentation = "https://docs.theredteam.io" -Repository = "https://github.com/RedTeamSubnet/challenge-template.git" -Issues = "https://github.com/RedTeamSubnet/challenge-template/issues" -Changelog = "https://github.com/RedTeamSubnet/challenge-template/blob/main/CHANGELOG.md" +Repository = "https://github.com/RedTeamSubnet/bot-virus-challenge.git" +Issues = "https://github.com/RedTeamSubnet/bot-virus-challenge/issues" +Changelog = "https://github.com/RedTeamSubnet/bot-virus-challenge/blob/main/CHANGELOG.md" diff --git a/requirements.txt b/requirements.txt index c22a8b5..5de82b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ python-dotenv>=1.0.1,<2.0.0 pydantic[email,timezone]>=2.0.3,<3.0.0 pydantic-settings>=2.2.1,<3.0.0 # redteam_core @ git+https://github.com/RedTeamSubnet/RedTeam.git@v4.2.2 +./requirements/rt_bv_score-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl +./requirements/vault_unlock-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl diff --git a/requirements/requirements.dev.txt b/requirements/requirements.dev.txt index 8bae40b..dddd559 100644 --- a/requirements/requirements.dev.txt +++ b/requirements/requirements.dev.txt @@ -1,5 +1,5 @@ # -e . # -r ./requirements.test.txt # -r ./requirements.build.txt -pyright>=1.1.392,<2.0.0 +pyright>=1.1.411,<2.0.0 pre-commit>=4.0.1,<5.0.0 diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index c674dc4..b88bfc9 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -26,7 +26,7 @@ fi ## --- Variables --- ## # Load from environment variables: -VERSION_FILE_PATH="${VERSION_FILE_PATH:-./VERSION.txt}" +VERSION_FILE_PATH="${VERSION_FILE_PATH:-./src/bv_challenge/__version__.py}" _BUMP_TYPE="" @@ -130,7 +130,7 @@ main() echo "[INFO]: Bumping version to '${_new_version}'..." # Update the version file with the new version: - echo "${_new_version}" > "${VERSION_FILE_PATH}" || exit 2 + echo -e "__version__ = \"${_new_version}\"" > "${VERSION_FILE_PATH}" || exit 2 echo "[OK]: New version: '${_new_version}'" ./scripts/sync-versions.sh -a || exit 2 diff --git a/scripts/get-version.sh b/scripts/get-version.sh index 2b59ebe..28bce9c 100755 --- a/scripts/get-version.sh +++ b/scripts/get-version.sh @@ -15,12 +15,12 @@ cd "${_PROJECT_DIR}" || exit 2 ## --- Variables --- ## # Load from environment variables: -VERSION_FILE_PATH="${VERSION_FILE_PATH:-./VERSION.txt}" +VERSION_FILE_PATH="${VERSION_FILE_PATH:-./src/bv_challenge/__version__.py}" ## --- Variables --- ## if [ -n "${VERSION_FILE_PATH}" ] && [ -f "${VERSION_FILE_PATH}" ]; then - _current_version=$(cat "${VERSION_FILE_PATH}") || exit 2 + _current_version=$(< "${VERSION_FILE_PATH}" grep "__version__ = " | awk -F' = ' '{print $2}' | tr -d '"') || exit 2 else _current_version="0.0.0" fi diff --git a/src/bot/Dockerfile b/src/bot/Dockerfile new file mode 100644 index 0000000..18533de --- /dev/null +++ b/src/bot/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 +# Reference bot image — two-file contract: this Dockerfile + bot.py. +FROM redteamsubnet61/bv-bot-base:latest + +WORKDIR /app + +# The base image ships Chrome and a Python venv (/opt/venv) on PATH, but not the +# Selenium Python client. Install it into that venv as root (the venv's +# site-packages is not writable by the default seluser). +USER root +RUN /opt/venv/bin/python3 -m pip install --no-cache-dir selenium + +COPY bot.py /app/bot.py + +# Drop back to the unprivileged default user to run the bot. +USER seluser + +ENTRYPOINT ["python3", "/app/bot.py"] diff --git a/src/bot/bot.py b/src/bot/bot.py new file mode 100644 index 0000000..3991edd --- /dev/null +++ b/src/bot/bot.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Reference bot for the Bot Virus challenge — single-file, two-file contract. + +The miner submission is exactly two files: this ``bot.py`` and a ``Dockerfile``. +This reference bot opens the challenge web page (``/_web``) with a headless +Chrome and waits for the browser-side SDK to collect the integrity signals and +POST the encrypted payload to ``/_eval``. It does NOT fill forms, move the +mouse, scroll, or submit anything from Python — submission must happen from the +browser context (``window.BV_SUBMITTED === true``). + +Endpoint configuration is read from the environment provided by the runner: + + CHALLENGE_WEB_URL e.g. http://challenge-api:10001/_web (preferred) + CHALLENGE_BASE_URL e.g. http://challenge-api:10001 (web url derived) + BV_SESSION_COUNT number of sessions to run (default: 1) + +Only the Selenium Python client is required on top of the base image. +""" + +import os +import sys +import logging +import subprocess +import tempfile +from urllib.parse import urlparse + +from selenium import webdriver +from selenium.common.exceptions import WebDriverException +from selenium.webdriver.common.by import By +from selenium.webdriver.remote.webdriver import WebDriver +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + + +logger = logging.getLogger(__name__) + +_VIEWPORT_WIDTH = 1440 +_VIEWPORT_HEIGHT = 900 +_DEFAULT_PORT = "10001" + + +def resolve_web_url() -> str: + """Resolve the challenge ``/_web`` URL from the environment. + + Order of preference: CHALLENGE_WEB_URL, then CHALLENGE_BASE_URL + "/_web", + then the container's default gateway host, then a sane default. + """ + + _web_url = os.getenv("CHALLENGE_WEB_URL") + if _web_url: + return _web_url + + _base_url = os.getenv("CHALLENGE_BASE_URL") + if _base_url: + return f"{_base_url.rstrip('/')}/_web" + + # Fallback: try to reach the host via the default gateway. + try: + _host = subprocess.check_output( + "ip route | awk '/default/ { print $3 }'", shell=True, text=True + ).strip() + except Exception: + _host = "challenge-api" + + _web_url = f"http://{_host}:{_DEFAULT_PORT}/_web" + logger.warning(f"CHALLENGE_WEB_URL not set, using fallback: {_web_url}") + return _web_url + + +def setup_driver(web_url: str) -> WebDriver: + """Initialize headless Chrome and load the challenge page.""" + + _options = webdriver.ChromeOptions() + _options.add_argument("--headless") + _options.add_argument("--no-sandbox") + _options.add_argument("--disable-gpu") + _options.add_argument("--ignore-certificate-errors") + + # Treat the (HTTP) challenge origin as secure so the SDK gets a secure + # context and WebCrypto SubtleCrypto is available for payload encryption. + # The flag takes an *origin* and only applies with a dedicated user-data-dir. + _parsed = urlparse(web_url) + _origin = f"{_parsed.scheme}://{_parsed.netloc}" + _options.add_argument(f"--unsafely-treat-insecure-origin-as-secure={_origin}") + _options.add_argument(f"--user-data-dir={tempfile.mkdtemp(prefix='bv-chrome-')}") + _options.add_argument(f"--window-size={_VIEWPORT_WIDTH},{_VIEWPORT_HEIGHT}") + + driver = webdriver.Chrome(options=_options) + driver.get(web_url) + + # Ensure the minimal verification page has loaded. + WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.ID, "status"))) + return driver + + +def run_session(driver: WebDriver) -> bool: + """Wait for the browser-side SDK to submit the payload to /_eval.""" + + try: + WebDriverWait(driver, 30).until( + lambda d: d.execute_script("return window.BV_SUBMITTED === true;") + ) + logger.info("Browser-side SDK submitted the payload to /_eval.") + + # Surface browser console logs for debugging (best effort). + try: + for entry in driver.get_log("browser"): + logger.info(f"[console][{entry.get('level')}] {entry.get('message')}") + except Exception as err: + logger.warning(f"Could not retrieve browser console logs: {err}") + + return True + except Exception as err: + logger.error(f"Browser-side submission did not complete: {err}") + return False + + +def automate(web_url: str) -> bool: + """Run a single automation session against the challenge web page.""" + + driver = None + try: + driver = setup_driver(web_url) + return run_session(driver) + except WebDriverException as err: + logger.error(f"WebDriver setup failed: {err}") + return False + except Exception as err: + logger.error(f"Automation failed: {err}") + return False + finally: + if driver is not None: + try: + driver.delete_all_cookies() + driver.execute_script("window.localStorage.clear();") + except Exception: + pass + driver.quit() + + +def main() -> None: + logging.basicConfig( + stream=sys.stdout, + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S %z", + format="[%(asctime)s | %(levelname)s | %(filename)s:%(lineno)d]: %(message)s", + ) + + logger.info("Starting WebUI automation bot...") + + web_url = resolve_web_url() + logger.info(f"Challenge web URL: {web_url}") + + try: + session_count = int(os.getenv("BV_SESSION_COUNT", "1")) + except (TypeError, ValueError): + session_count = 1 + session_count = max(1, session_count) + + logger.info(f"Running {session_count} session(s)") + + for index in range(session_count): + logger.info(f"Session {index + 1}/{session_count}") + automate(web_url) + + logger.info("Done!\n") + + +if __name__ == "__main__": + main() diff --git a/src/bv_challenge/challenge/Dockerfile b/src/bv_challenge/challenge/Dockerfile index c61f2c4..8018397 100644 --- a/src/bv_challenge/challenge/Dockerfile +++ b/src/bv_challenge/challenge/Dockerfile @@ -29,6 +29,7 @@ RUN --mount=type=cache,target=/root/.cache,sharing=locked \ # COPY ./requirements* ./ RUN --mount=type=cache,target=/root/.cache,sharing=locked \ --mount=type=bind,source=requirements.txt,target=requirements.txt \ + --mount=type=bind,source=requirements,target=requirements \ python3 -m uv pip install --prefix=/install -r ./requirements.txt @@ -140,6 +141,7 @@ FROM base AS app WORKDIR "${BV_CHALLENGE_API_DIR}" COPY --chown=${UID}:${GID} ./api ${BV_CHALLENGE_API_DIR}/api +COPY --chown=${UID}:${GID} ./templates ${BV_CHALLENGE_API_DIR}/templates COPY --chown=${UID}:${GID} --chmod=770 ./scripts/*.sh /usr/local/bin/ # VOLUME ["${BV_CHALLENGE_API_DATA_DIR}"] diff --git a/src/bv_challenge/challenge/api/__init__.py b/src/bv_challenge/challenge/api/__init__.py index 29c6809..5d91ce4 100644 --- a/src/bv_challenge/challenge/api/__init__.py +++ b/src/bv_challenge/challenge/api/__init__.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- + from api.__version__ import __version__ + __all__ = ["__version__"] diff --git a/src/bv_challenge/challenge/api/__main__.py b/src/bv_challenge/challenge/api/__main__.py index 4b0807c..4b27dc0 100644 --- a/src/bv_challenge/challenge/api/__main__.py +++ b/src/bv_challenge/challenge/api/__main__.py @@ -1,6 +1,25 @@ +# -*- coding: utf-8 -*- + +## Third-party libraries +from fastapi import FastAPI + +## Internal modules +from api.bootstrap import create_app, run_server from api.logger import logger -from api.main import main + + +app: FastAPI = create_app() + + +def main() -> None: + """Main function.""" + + run_server(app="api.__main__:app") + return + if __name__ == "__main__": - logger.info("Starting server from '__main__.py'...") + logger.info(f"Starting server from '__main__.py'...") main() + +__all__ = ["app"] diff --git a/src/bv_challenge/challenge/api/__version__.py b/src/bv_challenge/challenge/api/__version__.py index 6c8e6b9..a0235ce 100644 --- a/src/bv_challenge/challenge/api/__version__.py +++ b/src/bv_challenge/challenge/api/__version__.py @@ -1 +1 @@ -__version__ = "0.0.0" +__version__ = "0.0.2" \ No newline at end of file diff --git a/src/bv_challenge/challenge/api/bootstrap.py b/src/bv_challenge/challenge/api/bootstrap.py index ec71d87..436e770 100644 --- a/src/bv_challenge/challenge/api/bootstrap.py +++ b/src/bv_challenge/challenge/api/bootstrap.py @@ -1,17 +1,16 @@ -# Standard libraries -from typing import Any -from collections.abc import Callable +# -*- coding: utf-8 -*- -# Third-party libraries +## Standard libraries +import os +from typing import Union + +## Third-party libraries import uvicorn from uvicorn._types import ASGIApplication from pydantic import validate_call from fastapi import FastAPI -from beans_logging_fastapi import add_logger - -# Internal modules -from api.__version__ import __version__ +## Internal modules from api.config import config from api.lifespan import lifespan, pre_init from api.middleware import add_middlewares @@ -31,19 +30,13 @@ def create_app() -> FastAPI: pre_init() app = FastAPI( - title=config.api.title, - version=__version__, + title=config.api.name, + version=config.version, lifespan=lifespan, default_response_class=BaseResponse, **config.api.docs.model_dump(exclude={"enabled"}), ) - add_logger( - app=app, - config=config.api.logger, - has_proxy_headers=config.api.uvicorn.proxy_headers, - ) - add_middlewares(app=app) add_routers(app=app) add_mounts(app=app) @@ -53,21 +46,35 @@ def create_app() -> FastAPI: @validate_call(config={"arbitrary_types_allowed": True}) -def run_server(app: FastAPI | ASGIApplication | Callable[..., Any] | str) -> None: +def run_server(app: Union[ASGIApplication, str] = "main:app") -> None: """Run uvicorn server. Args: - app (FastAPI | - ASGIApplication | - Callable[..., Any] | - str , required): FastAPI application instance or ASGI application or import string. + app (Union[ASGIApplication, str], optional): ASGI application instance or module path. """ + _ssl_keyfile: Union[str, None] = None + _ssl_certfile: Union[str, None] = None + + if config.api.security.ssl.enabled: + _ssl_keyfile = os.path.join( + config.api.paths.ssl_dir, config.api.security.ssl.key_fname + ) + _ssl_certfile = os.path.join( + config.api.paths.ssl_dir, config.api.security.ssl.cert_fname + ) + uvicorn.run( app=app, host=config.api.bind_host, port=config.api.port, - **config.api.uvicorn.model_dump(), + access_log=False, + server_header=False, + proxy_headers=config.api.behind_proxy, + forwarded_allow_ips=config.api.security.forwarded_allow_ips, + ssl_keyfile=_ssl_keyfile, + ssl_certfile=_ssl_certfile, + **config.api.dev.model_dump(), ) return diff --git a/src/bv_challenge/challenge/api/config.py b/src/bv_challenge/challenge/api/config.py index 50e5cb6..c4045d4 100644 --- a/src/bv_challenge/challenge/api/config.py +++ b/src/bv_challenge/challenge/api/config.py @@ -1,46 +1,24 @@ -import os -from typing import TypeVar, Any +# -*- coding: utf-8 -*- -from pydantic import validate_call +import pathlib -from potato_util.io import read_all_configs +from onion_config import ConfigLoader +from beans_logging import logger -from api.core.constants import ENV_PREFIX_API, API_SLUG from api.core.configs import MainConfig -from api.logger import logger -ConfigType = TypeVar("ConfigType", bound=MainConfig) +config: MainConfig +try: + _parent_dir = pathlib.Path(__file__).parent.resolve() + _config_loader = ConfigLoader( + config_schema=MainConfig, configs_dirs=[str(_parent_dir / "configs")] + ) + # Main config object: + config: MainConfig = _config_loader.load() +except Exception: + logger.exception("Failed to load config:") + raise SystemExit(1) -@validate_call -def load_config( - configs_dir: str = os.path.join("/etc", API_SLUG), - env_name: str = f"{ENV_PREFIX_API}CONFIGS_DIR", - config_schema: type[ConfigType] = MainConfig, -) -> ConfigType: - _configs_dir_env = os.getenv(env_name, "") - if _configs_dir_env: - configs_dir = _configs_dir_env - _config_dict: dict[str, Any] = {} - if os.path.isdir(configs_dir): - _config_dict = read_all_configs(configs_dir=configs_dir) - - _config: ConfigType | None = None - try: - _config = config_schema(**_config_dict) - except Exception: - logger.exception("Failed to load config:") - raise SystemExit(1) - - return _config - - -config = load_config() - - -__all__ = [ - "MainConfig", - "load_config", - "config", -] +__all__ = ["config"] diff --git a/src/bv_challenge/challenge/api/configs/api.yml b/src/bv_challenge/challenge/api/configs/api.yml new file mode 100644 index 0000000..36b9eba --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/api.yml @@ -0,0 +1,18 @@ +env: "LOCAL" +debug: false + +api: + name: "Bot Virus Challenge" + slug: "rest-bv-challenge" + bind_host: "0.0.0.0" + port: 10001 + version: "1" + prefix: "" + gzip_min_size: 1024 # Bytes (1KB) + behind_proxy: true + behind_cf_proxy: true + dev: + reload: false + reload_includes: [".env", "*.json", "*.yml", "*.yaml", "*.md"] + reload_excludes: + [".*", "~*", ".py[cod]", ".sw.*", "__pycache__", "*.log", "logs"] diff --git a/src/bv_challenge/challenge/api/configs/challenge.yml b/src/bv_challenge/challenge/api/configs/challenge.yml new file mode 100644 index 0000000..74aa8ed --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/challenge.yml @@ -0,0 +1,52 @@ +challenge: + n_ch_per_epoch: 1 + n_run_per_ch: 10 + docker_ulimit: 32768 + allowed_pip_pkg_dt: "2025-01-01T00:00:00Z" + allowed_file_exts: [".py", ".json", ".yaml", ".yml", ".txt", ".pt"] + bot_timeout: 120 + # Layer 1/2 fallback score policy + gate_fail_score: 0.0 + metrics_processor_error_score: 0.5 + session_timeout_score: 0.0 + runner_fail_score: 0.0 + window_width: 1420 + window_height: 740 + n_checkboxes: 5 + cb_min_distance: 300 + cb_gen_max_factor: 20 + cb_size: 50 + cb_exclude_areas: [{ "x1": 500, "y1": 400, "x2": 950, "y2": 700 }] + cb_pre_action_list: + [ + { + "id": 0, + "type": "click", + "args": { "location": { "x": 510, "y": 58 } }, + }, + { + "id": 1, + "type": "click", + "args": { "location": { "x": 1079, "y": 92 } }, + }, + { + "id": 2, + "type": "click", + "args": { "location": { "x": 288, "y": 449 } }, + }, + { + "id": 3, + "type": "click", + "args": { "location": { "x": 909, "y": 350 } }, + }, + { + "id": 4, + "type": "click", + "args": { "location": { "x": 1314, "y": 289 } }, + }, + ] + + # VM configuration for remote Docker build/run + vm_endpoint: "http://bot-runner:8000" + vm_timeout: 120 + vm_ssl_verify: false diff --git a/src/bv_challenge/challenge/api/configs/docs.yml b/src/bv_challenge/challenge/api/configs/docs.yml new file mode 100644 index 0000000..b7fe3a1 --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/docs.yml @@ -0,0 +1,15 @@ +api: + docs: + enabled: true + openapi_url: "{api_prefix}/openapi.json" + docs_url: "{api_prefix}/docs" + redoc_url: "{api_prefix}/redoc" + swagger_ui_oauth2_redirect_url: "{api_prefix}/docs/oauth2-redirect" + summary: "This is the API documentation for the Bot Virus Challenge API." + openapi_tags: + - name: "Utils" + description: "Useful utility endpoints." + - name: "Challenge" + description: "Endpoints for challenge." + swagger_ui_parameters: + syntaxHighlight.theme: "nord" diff --git a/src/bv_challenge/challenge/api/configs/logger.yml b/src/bv_challenge/challenge/api/configs/logger.yml new file mode 100644 index 0000000..e74f4ee --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/logger.yml @@ -0,0 +1,52 @@ +logger: + app_name: "{api_slug}" + level: "INFO" + use_diagnose: false + stream: + format_str: "[{time:YYYY-MM-DD HH:mm:ss.SSS Z} | {level_short:<5} | {name}:{line}]: {message}" + # format_str: "[{time:YYYY-MM-DD HH:mm:ss.SSS Z:!UTC} | {level_short:<5} | {name}:{line}]: {message}" + std_handler: + enabled: true + file: + logs_dir: "../logs" + rotate_size: 10000000 # 10MB + rotate_time: "00:00:00" + backup_count: 90 + log_handlers: + enabled: true + format_str: "[{time:YYYY-MM-DD HH:mm:ss.SSS Z} | {level_short:<5} | {name}:{line}]: {message}" + # format_str: "[{time:YYYY-MM-DD HH:mm:ss.SSS Z:!UTC} | {level_short:<5} | {name}:{line}]: {message}" + log_path: "{app_name}.std.all.log" + err_path: "{app_name}.std.err.log" + json_handlers: + enabled: true + use_custom: false + log_path: "json/{app_name}.json.all.log" + err_path: "json/{app_name}.json.err.log" + intercept: + auto_load: + enabled: true + only_base: false + ignore_modules: [] + include_modules: [] + mute_modules: [ + "uvicorn.access", + # "uvicorn.error", + "multipart", + "watchfiles", + "watchfiles.main", + "watchfiles.watcher", + ] + extra: + http_std_msg_format: '[{request_id}] {client_host} {user_id} "{method} {url_path} HTTP/{http_version}" {status_code} {content_length}B {response_time}ms' + http_std_error_format: '[{request_id}] {client_host} {user_id} "{method} {url_path} HTTP/{http_version}" {status_code}' + http_std_debug_format: '[{request_id}] {client_host} {user_id} "{method} {url_path} HTTP/{http_version}"' + http_file_enabled: true + http_file_format: '{client_host} {request_id} {user_id} [{datetime}] "{method} {url_path} HTTP/{http_version}" {status_code} {content_length} "{h_referer}" "{h_user_agent}" {response_time}' + http_file_tz: "localtime" + # http_file_tz: "UTC" + http_log_path: "http/{app_name}.http.access.log" + http_err_path: "http/{app_name}.http.err.log" + http_json_enabled: true + http_json_path: "json.http/{app_name}.json.http.access.log" + http_json_err_path: "json.http/{app_name}.json.http.err.log" diff --git a/src/bv_challenge/challenge/api/configs/paths.yml b/src/bv_challenge/challenge/api/configs/paths.yml new file mode 100644 index 0000000..ea8c232 --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/paths.yml @@ -0,0 +1,8 @@ +api: + paths: + tmp_dir: "../tmp" + uploads_dir: "{tmp_dir}/uploads" + data_dir: "../data" + security_dir: "{data_dir}/security" + ssl_dir: "{data_dir}/security/ssl" + asymmetric_keys_dir: "{data_dir}/security/asymmetric_keys" diff --git a/src/bv_challenge/challenge/api/configs/security.yml b/src/bv_challenge/challenge/api/configs/security.yml new file mode 100644 index 0000000..57c6f72 --- /dev/null +++ b/src/bv_challenge/challenge/api/configs/security.yml @@ -0,0 +1,25 @@ +api: + security: + allowed_hosts: ["*"] + forwarded_allow_ips: ["*"] + cors: + allow_origins: ["*"] + allow_origin_regex: null + allow_headers: ["*"] + allow_methods: + ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "CONNECT"] + allow_credentials: false + expose_headers: [] + max_age: 600 # Seconds (10 minutes) + ssl: + enabled: false + generate: false + key_size: 2048 + key_fname: "key.pem" + cert_fname: "cert.pem" + asymmetric: + generate: false + algorithm: "RS256" + key_size: 2048 + private_key_fname: "private_key.pem" + public_key_fname: "public_key.pem" diff --git a/src/bv_challenge/challenge/api/core/configs/__init__.py b/src/bv_challenge/challenge/api/core/configs/__init__.py index 1c79b70..52c8168 100644 --- a/src/bv_challenge/challenge/api/core/configs/__init__.py +++ b/src/bv_challenge/challenge/api/core/configs/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * from ._main import * diff --git a/src/bv_challenge/challenge/api/core/configs/_api.py b/src/bv_challenge/challenge/api/core/configs/_api.py index 330dc39..03bab1d 100644 --- a/src/bv_challenge/challenge/api/core/configs/_api.py +++ b/src/bv_challenge/challenge/api/core/configs/_api.py @@ -1,148 +1,142 @@ +# -*- coding: utf-8 -*- + import sys -from typing import Any +from typing import Any, Dict -from pydantic import Field, field_validator, ValidationInfo, model_validator +from pydantic import Field, constr, field_validator, ValidationInfo, model_validator from pydantic_settings import SettingsConfigDict -from potato_util.constants import HTTPSchemeEnum - -from api.core.constants import ENV_PREFIX_API, API_SLUG -from api.core import utils - -from ._base import BaseConfig, FrozenBaseConfig -from ._uvicorn import UvicornConfig +from api.core.constants import ENV_PREFIX_API, HTTPSchemeEnum +from ._base import BaseConfig +from ._dev import DevConfig from ._security import SecurityConfig from ._docs import DocsConfig, FrozenDocsConfig from ._paths import PathsConfig, FrozenPathsConfig -from ._logger import LoggerConfigPM, FrozenLoggerConfigPM - - -class GZipConfig(FrozenBaseConfig): - minimum_size: int = Field(default=1024, ge=0, le=10_485_760) - compresslevel: int = Field(default=9, ge=1, le=9) - - model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX_API}GZIP_") class ApiConfig(BaseConfig): - title: str = Field( - default="Bot Virus Challenge", min_length=2, max_length=128 - ) - slug: str = Field(default=API_SLUG, min_length=2, max_length=128) + name: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=128) # type: ignore + slug: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=128) # type: ignore http_scheme: HTTPSchemeEnum = Field(default=HTTPSchemeEnum.http) - bind_host: str = Field( - default="0.0.0.0", min_length=2, max_length=128 # nosec B104 - ) - port: int = Field(default=10001, ge=80, lt=65536) - version: str = Field(default="1", min_length=1, max_length=16) - prefix: str = Field(default="", max_length=128) - gzip: GZipConfig = Field(default_factory=GZipConfig) - uvicorn: UvicornConfig = Field(default_factory=UvicornConfig) - security: SecurityConfig = Field(default_factory=SecurityConfig) - docs: DocsConfig = Field(default_factory=DocsConfig) - paths: PathsConfig = Field(default_factory=PathsConfig) - logger: LoggerConfigPM = Field(default_factory=LoggerConfigPM) - - @field_validator("prefix", mode="after") + bind_host: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=128) # type: ignore + port: int = Field(..., ge=80, lt=65536) + version: constr(strip_whitespace=True) = Field(..., min_length=1, max_length=16) # type: ignore + prefix: constr(strip_whitespace=True) = Field(..., max_length=128) # type: ignore + gzip_min_size: int = Field(..., ge=0, le=10_485_760) # 512 bytes + behind_proxy: bool = Field(...) + behind_cf_proxy: bool = Field(...) + dev: DevConfig = Field(...) + security: SecurityConfig = Field(...) + docs: DocsConfig = Field(...) + paths: PathsConfig = Field(...) + + @field_validator("slug") @classmethod - def _check_prefix(cls, val: str, info: ValidationInfo) -> str: - if ("version" in info.data) and val and ("{api_version}" in val): - val = val.format(api_version=info.data["version"]) + def _check_slug(cls, val: str, info: ValidationInfo) -> str: + if (not val) and ("name" in info.data): + val = ( + info.data["name"] + .lower() + .strip() + .replace(" ", "-") + .replace("_", "-") + .replace(".", "-") + ) return val - @field_validator("security", mode="after") + @field_validator("prefix") @classmethod - def _check_security( - cls, val: SecurityConfig, info: ValidationInfo - ) -> SecurityConfig: - if (not utils.is_running_bin()) and val.ssl.enabled: - info.data["http_scheme"] = HTTPSchemeEnum.https + def _check_prefix(cls, val: str, info: ValidationInfo) -> str: + if val and ("{api_version}" in val) and ("version" in info.data): + val = val.format(api_version=info.data["version"]) return val - @field_validator("docs", mode="after") + @field_validator("docs") @classmethod def _check_docs(cls, val: DocsConfig, info: ValidationInfo) -> DocsConfig: - _docs_dict = val.model_dump() - if ("prefix" in info.data) and val.enabled: - for _key, _doc in _docs_dict.items(): - if ( - isinstance(_doc, str) - and _key.endswith("url") - and ("{api_prefix}" in _doc) - ): - _docs_dict[_key] = _doc.format(api_prefix=info.data["prefix"]) - - val = FrozenDocsConfig(**_docs_dict) + if val.enabled and ("prefix" in info.data): + if val.openapi_url and ("{api_prefix}" in val.openapi_url): + val.openapi_url = val.openapi_url.format(api_prefix=info.data["prefix"]) + + if val.docs_url and ("{api_prefix}" in val.docs_url): + val.docs_url = val.docs_url.format(api_prefix=info.data["prefix"]) + + if val.redoc_url and ("{api_prefix}" in val.redoc_url): + val.redoc_url = val.redoc_url.format(api_prefix=info.data["prefix"]) + + if val.swagger_ui_oauth2_redirect_url and ( + "{api_prefix}" in val.swagger_ui_oauth2_redirect_url + ): + val.swagger_ui_oauth2_redirect_url = ( + val.swagger_ui_oauth2_redirect_url.format( + api_prefix=info.data["prefix"] + ) + ) + + val = FrozenDocsConfig(**val.model_dump()) return val - @field_validator("paths", mode="after") + @field_validator("paths") @classmethod def _check_paths(cls, val: PathsConfig, info: ValidationInfo) -> FrozenPathsConfig: - _paths_dict = val.model_dump() if "slug" in info.data: - for _key, _path in _paths_dict.items(): - if isinstance(_path, str) and ("{api_slug}" in _path): - _paths_dict[_key] = _path.format(api_slug=info.data["slug"]) + if "{api_slug}" in val.tmp_dir: + val.tmp_dir = val.tmp_dir.format(api_slug=info.data["slug"]) - val = FrozenPathsConfig(**_paths_dict) - return val + if "{api_slug}" in val.uploads_dir: + val.uploads_dir = val.uploads_dir.format(api_slug=info.data["slug"]) + elif "{tmp_dir}" in val.uploads_dir: + val.uploads_dir = val.uploads_dir.format(tmp_dir=val.tmp_dir) - @field_validator("logger", mode="after") - @classmethod - def _check_logger(cls, val: LoggerConfigPM, info: ValidationInfo) -> LoggerConfigPM: - if "slug" in info.data: - if "{api_slug}" in val.app_name: - val.app_name = val.app_name.format(api_slug=info.data["slug"]) - - if "{api_slug}" in val.file.logs_dir: - val.file.logs_dir = val.file.logs_dir.format(api_slug=info.data["slug"]) + if "{api_slug}" in val.data_dir: + val.data_dir = val.data_dir.format(api_slug=info.data["slug"]) - val = FrozenLoggerConfigPM(**val.model_dump()) + val = FrozenPathsConfig(**val.model_dump()) return val - model_config = SettingsConfigDict(env_prefix=ENV_PREFIX_API) - - -class FrozenApiConfig(ApiConfig): @model_validator(mode="before") @classmethod - def _check_args(cls, data: Any) -> Any: - if isinstance(data, dict) and utils.is_running_bin(): - _has_host_arg = False + def _check_args(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if ( + sys.argv[0].endswith("uvicorn") + or sys.argv[0].endswith("fastapi") + or sys.argv[0].endswith("gunicorn") + ): + _has_host = False for _i, _arg in enumerate(sys.argv): - if ( - _arg.startswith("--ssl") - or _arg.startswith("--keyfile") - or _arg.startswith("--certfile") - ): - data["http_scheme"] = HTTPSchemeEnum.https + if _arg.startswith("--ssl"): + values["http_scheme"] = HTTPSchemeEnum.https if _arg.startswith("--host="): - _has_host_arg = True - data["bind_host"] = _arg.split("=")[1] + _has_host = True + values["bind_host"] = _arg.split("=")[1] elif (_arg == "--host") and (_i + 1 < len(sys.argv)): - _has_host_arg = True - data["bind_host"] = sys.argv[_i + 1] + _has_host = True + values["bind_host"] = sys.argv[_i + 1] if _arg.startswith("--port="): - data["port"] = int(_arg.split("=")[1]) + values["port"] = int(_arg.split("=")[1]) elif (_arg == "--port") and (_i + 1 < len(sys.argv)): - data["port"] = int(sys.argv[_i + 1]) + values["port"] = int(sys.argv[_i + 1]) - if not _has_host_arg: - data["bind_host"] = "127.0.0.1" + if not _has_host: + values["bind_host"] = "127.0.0.1" if sys.argv[0].endswith("fastapi") and sys.argv[1] == "run": - data["bind_host"] = "0.0.0.0" # nosec B104 + values["bind_host"] = "0.0.0.0" - return data + elif values["security"]["ssl"]["enabled"]: + values["http_scheme"] = HTTPSchemeEnum.https + return values + + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX_API) + + +class FrozenApiConfig(ApiConfig): model_config = SettingsConfigDict(frozen=True) -__all__ = [ - "ApiConfig", - "FrozenApiConfig", -] +__all__ = ["ApiConfig", "FrozenApiConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_base.py b/src/bv_challenge/challenge/api/core/configs/_base.py index ed31b1f..cb9fb83 100644 --- a/src/bv_challenge/challenge/api/core/configs/_base.py +++ b/src/bv_challenge/challenge/api/core/configs/_base.py @@ -1,70 +1,31 @@ +# -*- coding: utf-8 -*- + +from typing import Type, Tuple + from pydantic_settings import ( BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource, - CliSettingsSource, - NestedSecretsSettingsSource, ) -from api.core import utils - class BaseConfig(BaseSettings): - model_config = SettingsConfigDict( - extra="allow", - env_file=".env", - validate_default=True, - validate_assignment=True, - arbitrary_types_allowed=True, - ) - - -class FrozenBaseConfig(BaseConfig): - model_config = SettingsConfigDict(frozen=True) + model_config = SettingsConfigDict(extra="allow", arbitrary_types_allowed=True) @classmethod def settings_customise_sources( cls, - settings_cls: type[BaseSettings], + settings_cls: Type[BaseSettings], init_settings: PydanticBaseSettingsSource, env_settings: PydanticBaseSettingsSource, dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - return ( - NestedSecretsSettingsSource(file_secret_settings), - dotenv_settings, - env_settings, - init_settings, - ) + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return dotenv_settings, env_settings, init_settings, file_secret_settings -class BaseMainConfig(FrozenBaseConfig): - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - _sources = [] - if not utils.is_running_bin(): - _sources.append(CliSettingsSource(settings_cls, cli_parse_args=True)) - _sources.extend( - [ - NestedSecretsSettingsSource(file_secret_settings), - dotenv_settings, - env_settings, - init_settings, - ] - ) - return tuple(_sources) +class FrozenBaseConfig(BaseConfig): + model_config = SettingsConfigDict(frozen=True) -__all__ = [ - "BaseConfig", - "FrozenBaseConfig", - "BaseMainConfig", -] +__all__ = ["BaseConfig", "FrozenBaseConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_challenge.py b/src/bv_challenge/challenge/api/core/configs/_challenge.py new file mode 100644 index 0000000..58f5bf1 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/configs/_challenge.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +from datetime import datetime +from typing import List, Optional, Dict, Union + +from pydantic import Field, constr +from pydantic_settings import SettingsConfigDict + +from api.core.constants import ALPHANUM_HOST_REGEX, ENV_PREFIX +from ._base import FrozenBaseConfig + + +class ChallengeConfig(FrozenBaseConfig): + n_ch_per_epoch: int = Field(...) + n_run_per_ch: int = Field(...) + docker_ulimit: int = Field(...) + allowed_pip_pkg_dt: datetime = Field(...) + allowed_file_exts: List[ + constr( + strip_whitespace=True, + min_length=2, + max_length=16, + pattern=ALPHANUM_HOST_REGEX, + ) # type: ignore + ] = Field(..., min_length=1) + bot_timeout: int = Field(..., ge=1) + # Layer 1/2 fallback score policy (not detector secrets — just policy). + gate_fail_score: float = Field(default=0.0, ge=0.0, le=1.0) + metrics_processor_error_score: float = Field(default=0.5, ge=0.0, le=1.0) + session_timeout_score: float = Field(default=0.0, ge=0.0, le=1.0) + runner_fail_score: float = Field(default=0.0, ge=0.0, le=1.0) + window_width: int = Field(..., ge=20, le=12000) + window_height: int = Field(..., ge=20, le=12000) + n_checkboxes: int = Field(..., ge=2, le=100) + cb_min_distance: int = Field(..., ge=1, le=1000) + cb_gen_max_factor: int = Field(..., ge=2, le=100) + cb_size: int = Field(..., ge=10, le=100) + cb_exclude_areas: Optional[List[Dict[str, int]]] = Field(default=None) + cb_pre_action_list: Optional[ + List[Dict[str, Union[int, str, Dict[str, Dict[str, int]]]]] + ] = Field(default=None) + # VM configuration for remote Docker build/run + vm_endpoint: str = Field(...) + vm_timeout: int = Field(default=120, ge=1) + vm_ssl_verify: bool = Field(default=True) + + model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX}CHALLENGE_") + + +__all__ = ["ChallengeConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_dev.py b/src/bv_challenge/challenge/api/core/configs/_dev.py new file mode 100644 index 0000000..2012d55 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/configs/_dev.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from typing import Any, Dict, List, Optional + +from pydantic import Field, constr, model_validator +from pydantic_settings import SettingsConfigDict + +from api.core.constants import ENV_PREFIX_API +from ._base import BaseConfig + + +class DevConfig(BaseConfig): + reload: bool = Field(...) + reload_includes: Optional[ + List[constr(strip_whitespace=True, min_length=1, max_length=256)] # type: ignore + ] = Field(default=None) + reload_excludes: Optional[ + List[constr(strip_whitespace=True, min_length=1, max_length=256)] # type: ignore + ] = Field(default=None) + + model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX_API}DEV_") + + +class FrozenDevConfig(DevConfig): + @model_validator(mode="before") + @classmethod + def _check_all(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if not values["reload"]: + values["reload_includes"] = None + values["reload_excludes"] = None + + return values + + model_config = SettingsConfigDict(frozen=True) + + +__all__ = ["DevConfig", "FrozenDevConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_docs.py b/src/bv_challenge/challenge/api/core/configs/_docs.py index 1fa3484..1db5ba5 100644 --- a/src/bv_challenge/challenge/api/core/configs/_docs.py +++ b/src/bv_challenge/challenge/api/core/configs/_docs.py @@ -1,84 +1,69 @@ -from typing import Any +# -*- coding: utf-8 -*- -from pydantic import Field, model_validator -from pydantic_settings import SettingsConfigDict +from typing import Any, Dict, List, Optional -from potato_util import validator +from pydantic import Field, constr, model_validator +from pydantic_settings import SettingsConfigDict from api.core.constants import ENV_PREFIX_API - +from api.core.utils import validator from ._base import BaseConfig class DocsConfig(BaseConfig): - enabled: bool = Field(default=True) - openapi_url: str | None = Field(default="{api_prefix}/openapi.json") - docs_url: str | None = Field(default="{api_prefix}/docs") - redoc_url: str | None = Field(default="{api_prefix}/redoc") - swagger_ui_oauth2_redirect_url: str | None = Field( - default="{api_prefix}/docs/oauth2-redirect" - ) - summary: str | None = Field(default="This is a RedTeam Subnet's bot virus challenge repository.") + enabled: bool = Field(...) + openapi_url: Optional[ + constr(strip_whitespace=True, max_length=128) # type: ignore + ] = Field(default=None) + docs_url: Optional[ + constr(strip_whitespace=True, max_length=128) # type: ignore + ] = Field(default=None) + redoc_url: Optional[ + constr(strip_whitespace=True, max_length=128) # type: ignore + ] = Field(default=None) + swagger_ui_oauth2_redirect_url: Optional[ + constr(strip_whitespace=True, max_length=128) # type: ignore + ] = Field(default=None) + summary: Optional[ + constr(strip_whitespace=True, min_length=2, max_length=128) # type: ignore + ] = Field(default=None) description: str = Field(default="", max_length=8192) - terms_of_service: str | None = Field( - default="https://theredteam.io/terms" - ) - contact: dict[str, Any] | None = Field( - default={ - "name": "Support Team", - "email": "support@theredteam.io", - "url": "https://theredteam.io/contact", - } - ) - license_info: dict[str, Any] | None = Field( - default={ - "name": "MIT License", - "url": "https://opensource.org/licenses/mit", - } - ) - openapi_tags: list[dict[str, Any]] | None = Field( - default=[ - {"name": "Utils", "description": "Useful utility endpoints."}, - {"name": "Challenge", "description": "Endpoints for challenge."}, - {"name": "Default", "description": "Redirection of default endpoints."}, - ] - ) - swagger_ui_parameters: dict[str, Any] | None = Field( - default={"syntaxHighlight": {"theme": "nord"}} - ) + terms_of_service: Optional[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = Field(default=None) + contact: Optional[Dict[str, Any]] = Field(default=None) + license_info: Optional[Dict[str, Any]] = Field(default=None) + openapi_tags: Optional[List[Dict[str, Any]]] = Field(default=None) + swagger_ui_parameters: Optional[Dict[str, Any]] = Field(default=None) model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX_API}DOCS_") class FrozenDocsConfig(DocsConfig): + @model_validator(mode="before") @classmethod - def _check_all(cls, data: Any) -> Any: - if isinstance(data, dict): - if ("openapi_url" in data) and (data["openapi_url"] == ""): - data["openapi_url"] = None - - if ("docs_url" in data) and (data["docs_url"] == ""): - data["docs_url"] = None - - if ("redoc_url" in data) and (data["redoc_url"] == ""): - data["redoc_url"] = None - - if ("swagger_ui_oauth2_redirect_url" in data) and ( - data["swagger_ui_oauth2_redirect_url"] == "" - ): - data["swagger_ui_oauth2_redirect_url"] = None - - try: - if ("enabled" in data) and validator.is_falsy(data["enabled"]): - data["openapi_url"] = None - data["docs_url"] = None - data["redoc_url"] = None - data["swagger_ui_oauth2_redirect_url"] = None - except ValueError: - pass - - return data + def _check_all(cls, values: Dict[str, Any]) -> Dict[str, Any]: + + if values["openapi_url"] == "": + values["openapi_url"] = None + + if values["docs_url"] == "": + values["docs_url"] = None + + if values["redoc_url"] == "": + values["redoc_url"] = None + + if values["swagger_ui_oauth2_redirect_url"] == "": + values["swagger_ui_oauth2_redirect_url"] = None + + if validator.is_falsy(values["enabled"]): + values["openapi_url"] = None + values["docs_url"] = None + values["redoc_url"] = None + values["swagger_ui_oauth2_redirect_url"] = None + + return values model_config = SettingsConfigDict(frozen=True) diff --git a/src/bv_challenge/challenge/api/core/configs/_logger.py b/src/bv_challenge/challenge/api/core/configs/_logger.py deleted file mode 100644 index 603e579..0000000 --- a/src/bv_challenge/challenge/api/core/configs/_logger.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -from pydantic import Field, field_validator -from pydantic_settings import SettingsConfigDict - -from beans_logging.config import FileConfigPM as BaseFileConfigPM -from beans_logging_fastapi import LoggerConfigPM as BaseLoggerConfigPM - -from api.core.constants import ENV_PREFIX_API - -from ._base import BaseConfig - - -class FileConfigPM(BaseFileConfigPM, BaseConfig): - logs_dir: str = Field(default="./logs", min_length=2, max_length=1024) - - @field_validator("logs_dir", mode="after") - @classmethod - def _check_logger(cls, val: str) -> str: - _logs_dir = os.getenv(f"{ENV_PREFIX_API}LOGS_DIR", "") - if _logs_dir: - val = _logs_dir - - return val - - -class LoggerConfigPM(BaseLoggerConfigPM, BaseConfig): - app_name: str = Field(default="{api_slug}", min_length=1, max_length=128) - file: FileConfigPM = Field(default_factory=FileConfigPM) # type: ignore - - model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX_API}LOGGER_") - - -class FrozenLoggerConfigPM(LoggerConfigPM): - model_config = SettingsConfigDict(frozen=True) - - -__all__ = [ - "FileConfigPM", - "LoggerConfigPM", - "FrozenLoggerConfigPM", -] diff --git a/src/bv_challenge/challenge/api/core/configs/_main.py b/src/bv_challenge/challenge/api/core/configs/_main.py index c699916..0f33538 100644 --- a/src/bv_challenge/challenge/api/core/configs/_main.py +++ b/src/bv_challenge/challenge/api/core/configs/_main.py @@ -1,56 +1,99 @@ +# -*- coding: utf-8 -*- + import os +from typing_extensions import Self -from pydantic import Field, field_validator, ValidationInfo +from pydantic import Field, constr, field_validator, ValidationInfo, model_validator from pydantic_settings import SettingsConfigDict -from potato_util.constants import EnvEnum - -from api.core.constants import ENV_PREFIX +from beans_logging import LoggerConfigPM -from ._base import BaseMainConfig -from ._uvicorn import UvicornConfig, FrozenUvicornConfig +from api.__version__ import __version__ +from api.core.constants import EnvEnum, ENV_PREFIX, ENV_PREFIX_API +from ._base import FrozenBaseConfig +from ._dev import DevConfig, FrozenDevConfig from ._api import ApiConfig, FrozenApiConfig +from ._challenge import ChallengeConfig # Main config schema: -class MainConfig(BaseMainConfig): - env: EnvEnum = Field(default=EnvEnum.LOCAL, alias="env") - debug: bool = Field(default=False, alias="debug") - api: ApiConfig = Field(default_factory=ApiConfig) +class MainConfig(FrozenBaseConfig): + env: EnvEnum = Field(...) + debug: bool = Field(...) + version: constr(strip_whitespace=True) = Field( # type: ignore + default=__version__, min_length=3, max_length=32 + ) + api: ApiConfig = Field(...) + challenge: ChallengeConfig = Field(...) + logger: LoggerConfigPM = Field(default_factory=LoggerConfigPM) + + @field_validator("env") + @classmethod + def _check_env(cls, val: EnvEnum) -> EnvEnum: + _env = "ENV" + if _env in os.environ: + _env = os.getenv(_env).upper() + val = EnvEnum(_env) + + return val + + @field_validator("debug") + @classmethod + def _check_debug(cls, val: str) -> str: + _debug_env = "DEBUG" + if _debug_env in os.environ: + val = os.getenv(_debug_env) + + return val - @field_validator("api", mode="after") + @field_validator("version") + @classmethod + def _check_version(cls, val: str) -> str: + val = __version__ + return val + + @field_validator("api") @classmethod def _check_api(cls, val: ApiConfig, info: ValidationInfo) -> FrozenApiConfig: - _uvicorn: UvicornConfig = val.uvicorn + _dev: DevConfig = val.dev if ("env" in info.data) and (info.data["env"] == EnvEnum.DEVELOPMENT): - _uvicorn.reload = True + _dev.reload = True - if val.security.ssl.enabled: - if not _uvicorn.ssl_keyfile: - _uvicorn.ssl_keyfile = os.path.join( - val.paths.ssl_dir, val.security.ssl.key_fname - ) + _dev = FrozenDevConfig(**_dev.model_dump()) + val = FrozenApiConfig(dev=_dev, **val.model_dump(exclude={"dev"})) + return val - if not _uvicorn.ssl_certfile: - _uvicorn.ssl_certfile = os.path.join( - val.paths.ssl_dir, val.security.ssl.cert_fname - ) + @field_validator("logger") + @classmethod + def _check_logger(cls, val: LoggerConfigPM, info: ValidationInfo) -> LoggerConfigPM: + if "api" in info.data: + if not val.app_name: + val.app_name = info.data["api"].slug + elif "{api_slug}" in val.app_name: + val.app_name = val.app_name.format(api_slug=info.data["api"].slug) + + _logs_dir_env = f"{ENV_PREFIX_API}LOGS_DIR" + if _logs_dir_env in os.environ: + val.file.logs_dir = os.getenv(_logs_dir_env) - _uvicorn = FrozenUvicornConfig(**_uvicorn.model_dump()) - val = FrozenApiConfig(uvicorn=_uvicorn, **val.model_dump(exclude={"uvicorn"})) return val - model_config = SettingsConfigDict( - env_prefix=ENV_PREFIX, - env_nested_delimiter="__", - cli_prefix="", - secrets_dir="/run/secrets", - secrets_prefix="", - secrets_nested_delimiter="_", - secrets_dir_missing="ok", # pragma: allowlist secret - ) # type: ignore + @model_validator(mode="after") + def _check_required_envs(self) -> Self: + _required_envs = [ + # f"{ENV_PREFIX_API}SECURITY_JWT_SECRET", + ] + + if (self.env == EnvEnum.STAGING) or (self.env == EnvEnum.PRODUCTION): + for _required_env in _required_envs: + if _required_env not in os.environ: + raise ValueError( + f"Missing required '{_required_env}' environment variable for STAGING/PRODUCTION environment!" + ) + + return self + + model_config = SettingsConfigDict(env_prefix=ENV_PREFIX, env_nested_delimiter="__") -__all__ = [ - "MainConfig", -] +__all__ = ["MainConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_paths.py b/src/bv_challenge/challenge/api/core/configs/_paths.py index b880b79..30da454 100644 --- a/src/bv_challenge/challenge/api/core/configs/_paths.py +++ b/src/bv_challenge/challenge/api/core/configs/_paths.py @@ -1,43 +1,50 @@ +# -*- coding: utf-8 -*- + import os -from typing import Any +from typing import Any, Dict -from pydantic import Field, model_validator, field_validator +from pydantic import Field, constr, model_validator, field_validator from pydantic_settings import SettingsConfigDict from api.core.constants import ENV_PREFIX_API - from ._base import BaseConfig class PathsConfig(BaseConfig): - tmp_dir: str = Field(default="./tmp", min_length=2, max_length=1024) # nosec B108 - uploads_dir: str = Field(default="{tmp_dir}/uploads", min_length=2, max_length=1024) - data_dir: str = Field(default="./data", min_length=2, max_length=1024) - security_dir: str = Field( - default="{data_dir}/security", min_length=2, max_length=1024 + tmp_dir: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=1024) # type: ignore + uploads_dir: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=1024 ) - ssl_dir: str = Field( - default="{data_dir}/security/ssl", min_length=2, max_length=1024 + data_dir: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=1024 ) - asymmetric_keys_dir: str = Field( - default="{data_dir}/security/asymmetric_keys", min_length=2, max_length=1024 + security_dir: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=1024 ) + ssl_dir: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=1024) # type: ignore + asymmetric_keys_dir: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=1024 + ) + # models_dir: constr(strip_whitespace=True) = Field( # type: ignore + # ..., min_length=2, max_length=1024 + # ) + # model_dir: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=1024) # type: ignore - @field_validator("tmp_dir", mode="after") + @field_validator("data_dir") @classmethod - def _check_tmp_dir(cls, val: str) -> str: - _tmp_dir = os.getenv(f"{ENV_PREFIX_API}TMP_DIR", "") - if _tmp_dir: - val = _tmp_dir + def _check_data_dir(cls, val: str) -> str: + _data_dir_env = f"{ENV_PREFIX_API}DATA_DIR" + if _data_dir_env in os.environ: + val = os.getenv(_data_dir_env) return val - @field_validator("data_dir", mode="after") + @field_validator("tmp_dir") @classmethod - def _check_data_dir(cls, val: str) -> str: - _data_dir = os.getenv(f"{ENV_PREFIX_API}DATA_DIR", "") - if _data_dir: - val = _data_dir + def _check_tmp_dir(cls, val: str) -> str: + _tmp_dir_env = f"{ENV_PREFIX_API}TMP_DIR" + if _tmp_dir_env in os.environ: + val = os.getenv(_tmp_dir_env) return val @@ -47,22 +54,14 @@ def _check_data_dir(cls, val: str) -> str: class FrozenPathsConfig(PathsConfig): @model_validator(mode="before") @classmethod - def _check_all(cls, data: Any) -> Any: - if isinstance(data, dict): - for _key, _val in data.items(): - if isinstance(_val, str): - if ("data_dir" in data) and ("{data_dir}" in _val): - data[_key] = _val.format(data_dir=data["data_dir"]) - - if ("tmp_dir" in data) and ("{tmp_dir}" in _val): - data[_key] = _val.format(tmp_dir=data["tmp_dir"]) + def _check_all(cls, values: Dict[str, Any]) -> Dict[str, Any]: + for _key, _val in values.items(): + if isinstance(_val, str) and ("{data_dir}" in _val): + values[_key] = _val.format(data_dir=values["data_dir"]) - return data + return values model_config = SettingsConfigDict(frozen=True) -__all__ = [ - "PathsConfig", - "FrozenPathsConfig", -] +__all__ = ["PathsConfig", "FrozenPathsConfig"] diff --git a/src/bv_challenge/challenge/api/core/configs/_security.py b/src/bv_challenge/challenge/api/core/configs/_security.py index fdb16b6..f747b28 100644 --- a/src/bv_challenge/challenge/api/core/configs/_security.py +++ b/src/bv_challenge/challenge/api/core/configs/_security.py @@ -1,53 +1,51 @@ -from pydantic import Field, constr, SecretStr +# -*- coding: utf-8 -*- + +from typing import List, Optional + +from pydantic import Field, constr from pydantic_settings import SettingsConfigDict -from potato_util.constants import ( +from api.core.constants import ( + ENV_PREFIX_API, HTTP_METHOD_REGEX, ASYMMETRIC_ALGORITHM_REGEX, - JWT_ALGORITHM_REGEX, ) - -from api.core.constants import ENV_PREFIX, ENV_PREFIX_API - from ._base import FrozenBaseConfig + _ENV_PREFIX_SECURITY = f"{ENV_PREFIX_API}SECURITY_" class CorsConfig(FrozenBaseConfig): - allow_origins: list[str] = Field(default=["*"]) - allow_origin_regex: str | None = Field(default=None) - allow_headers: list[str] = Field(default=["*"]) - allow_methods: list[constr(strip_whitespace=True, pattern=HTTP_METHOD_REGEX)] = ( # type: ignore - Field( - default=[ - "GET", - "POST", - "PUT", - "PATCH", - "DELETE", - "HEAD", - "OPTIONS", - "CONNECT", - ] - ) + allow_origins: List[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = Field(...) + allow_origin_regex: Optional[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = Field(default=None) + allow_headers: List[ + constr(strip_whitespace=True, min_length=1, max_length=128) # type: ignore + ] = Field(...) + allow_methods: List[constr(strip_whitespace=True, pattern=HTTP_METHOD_REGEX)] = ( # type: ignore + Field(...) ) - allow_credentials: bool = Field(default=False) - allow_private_network: bool = Field(default=False) - expose_headers: list[str] = Field(default=[]) - max_age: int = Field(default=600, ge=0, le=86_400) # Seconds (10 minutes) + allow_credentials: bool = Field(...) + expose_headers: List[ + constr(strip_whitespace=True, min_length=1, max_length=128) # type: ignore + ] = Field(...) + max_age: int = Field(..., ge=0, le=86_400) model_config = SettingsConfigDict(env_prefix=f"{_ENV_PREFIX_SECURITY}CORS_") class X509AttrsConfig(FrozenBaseConfig): - C: str = Field(default="US", min_length=2, max_length=2) - ST: str = Field(default="Washington", min_length=2, max_length=256) - L: str = Field(default="Seattle", min_length=2, max_length=256) - O: str = Field(default="Organization", min_length=2, max_length=256) - OU: str = Field(default="Organization Unit", min_length=2, max_length=256) - CN: str = Field(default="localhost", min_length=2, max_length=256) - DNS: str = Field(default="localhost", min_length=2, max_length=256) + C: constr(strip_whitespace=True, to_upper=True) = Field(default="US", min_length=2, max_length=2) # type: ignore + ST: constr(strip_whitespace=True) = Field(default="Washington", min_length=2, max_length=256) # type: ignore + L: constr(strip_whitespace=True) = Field(default="Seattle", min_length=2, max_length=256) # type: ignore + O: constr(strip_whitespace=True) = Field(default="Organization", min_length=2, max_length=256) # type: ignore + OU: constr(strip_whitespace=True) = Field(default="Organization Unit", min_length=2, max_length=256) # type: ignore + CN: constr(strip_whitespace=True) = Field(default="localhost", min_length=2, max_length=256) # type: ignore + DNS: constr(strip_whitespace=True) = Field(default="localhost", min_length=2, max_length=256) # type: ignore model_config = SettingsConfigDict( env_prefix=f"{_ENV_PREFIX_SECURITY}SSL_X509_ATTRS_" @@ -55,60 +53,40 @@ class X509AttrsConfig(FrozenBaseConfig): class SSLConfig(FrozenBaseConfig): - enabled: bool = Field(default=False) - generate: bool = Field(default=False) - key_size: int = Field(default=2048, ge=2048, le=8192) - key_fname: str = Field(default="key.pem", min_length=2, max_length=256) - cert_fname: str = Field(default="cert.pem", min_length=2, max_length=256) + enabled: bool = Field(...) + generate: bool = Field(...) + key_size: int = Field(..., ge=2048, le=8192) + key_fname: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=256) # type: ignore + cert_fname: constr(strip_whitespace=True) = Field(..., min_length=2, max_length=256) # type: ignore x509_attrs: X509AttrsConfig = Field(default_factory=X509AttrsConfig) model_config = SettingsConfigDict(env_prefix=f"{_ENV_PREFIX_SECURITY}SSL_") class AsymmetricConfig(FrozenBaseConfig): - generate: bool = Field(default=False) - algorithm: str = Field(default="RS256", pattern=ASYMMETRIC_ALGORITHM_REGEX) - key_size: int = Field(default=2048, ge=2048, le=8192) - private_key_fname: str = Field( - default="private_key.pem", min_length=2, max_length=256 + generate: bool = Field(...) + algorithm: constr(strip_whitespace=True) = Field(..., pattern=ASYMMETRIC_ALGORITHM_REGEX) # type: ignore + key_size: int = Field(..., ge=2048, le=8192) + private_key_fname: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=256 ) - public_key_fname: str = Field( - default="public_key.pem", min_length=2, max_length=256 + public_key_fname: constr(strip_whitespace=True) = Field( # type: ignore + ..., min_length=2, max_length=256 ) model_config = SettingsConfigDict(env_prefix=f"{_ENV_PREFIX_SECURITY}ASYMMETRIC_") -class JWTConfig(FrozenBaseConfig): - secret: SecretStr = Field( - default_factory=lambda: SecretStr(f"{ENV_PREFIX}JWT_SECRET123"), - min_length=8, - max_length=64, - ) - algorithm: str = Field(default="HS256", pattern=JWT_ALGORITHM_REGEX) - - model_config = SettingsConfigDict(env_prefix=f"{_ENV_PREFIX_SECURITY}JWT_") - - -class PasswordConfig(FrozenBaseConfig): - pepper: SecretStr = Field( - default_factory=lambda: SecretStr(f"{ENV_PREFIX}PASSWORD_PEPPER123"), - min_length=8, - max_length=32, - ) - min_length: int = Field(default=8, ge=8, le=128) - max_length: int = Field(default=128, ge=8, le=128) - - model_config = SettingsConfigDict(env_prefix=f"{_ENV_PREFIX_SECURITY}PASSWORD_") - - class SecurityConfig(FrozenBaseConfig): - allowed_hosts: list[str] = Field(default=["*"]) - cors: CorsConfig = Field(default_factory=CorsConfig) - ssl: SSLConfig = Field(default_factory=SSLConfig) - asymmetric: AsymmetricConfig = Field(default_factory=AsymmetricConfig) - jwt: JWTConfig = Field(default_factory=JWTConfig) - password: PasswordConfig = Field(default_factory=PasswordConfig) + allowed_hosts: List[constr(strip_whitespace=True, min_length=1, max_length=256)] = ( # type: ignore + Field(...) + ) + forwarded_allow_ips: List[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = Field(...) + cors: CorsConfig = Field(...) + ssl: SSLConfig = Field(...) + asymmetric: AsymmetricConfig = Field(...) model_config = SettingsConfigDict(env_prefix=_ENV_PREFIX_SECURITY) @@ -119,6 +97,4 @@ class SecurityConfig(FrozenBaseConfig): "X509AttrsConfig", "SSLConfig", "AsymmetricConfig", - "JWTConfig", - "PasswordConfig", ] diff --git a/src/bv_challenge/challenge/api/core/configs/_uvicorn.py b/src/bv_challenge/challenge/api/core/configs/_uvicorn.py deleted file mode 100644 index 223faa7..0000000 --- a/src/bv_challenge/challenge/api/core/configs/_uvicorn.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -from typing import Any - -from pydantic import Field, model_validator -from pydantic_settings import SettingsConfigDict - -from api.core.constants import ENV_PREFIX_API - -from ._base import BaseConfig - - -class UvicornConfig(BaseConfig): - access_log: bool = Field(default=False) - server_header: bool = Field(default=False) - proxy_headers: bool = Field(default=True) - forwarded_allow_ips: list[str] | str | None = Field(default=["*"]) - ssl_keyfile: str | None = Field(default=None) - ssl_certfile: str | None = Field(default=None) - reload: bool = Field(default=False) - reload_dirs: list[str] | str | None = Field(default=None) - reload_includes: list[str] | str | None = Field( - default=["*.json", "*.yml", "*.yaml", "*.toml", "*.md"] - ) - reload_excludes: list[str] | str | None = Field( - default=[".*", "~*", ".py[cod]", ".sw.*", "__pycache__", "*.log", "logs"] - ) - - model_config = SettingsConfigDict(env_prefix=f"{ENV_PREFIX_API}UVICORN_") - - -class FrozenUvicornConfig(UvicornConfig): - @model_validator(mode="before") - @classmethod - def _check_all(cls, data: Any) -> Any: - if isinstance(data, dict): - if "reload" in data: - if data["reload"]: - if (not data.get("reload_dirs")) and os.path.isdir("./src"): - data["reload_dirs"] = ["./src"] - else: - data["reload_includes"] = None - data["reload_excludes"] = None - data["reload_dirs"] = None - - return data - - model_config = SettingsConfigDict(frozen=True) - - -__all__ = [ - "UvicornConfig", - "FrozenUvicornConfig", -] diff --git a/src/bv_challenge/challenge/api/core/constants/__init__.py b/src/bv_challenge/challenge/api/core/constants/__init__.py index f6ecab7..8bd1879 100644 --- a/src/bv_challenge/challenge/api/core/constants/__init__.py +++ b/src/bv_challenge/challenge/api/core/constants/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * from ._regex import * diff --git a/src/bv_challenge/challenge/api/core/constants/_base.py b/src/bv_challenge/challenge/api/core/constants/_base.py index ec418c1..fbe5f32 100644 --- a/src/bv_challenge/challenge/api/core/constants/_base.py +++ b/src/bv_challenge/challenge/api/core/constants/_base.py @@ -1,10 +1,62 @@ -ENV_PREFIX = "BV_CHALLENGE_" -ENV_PREFIX_API = f"{ENV_PREFIX}API_" +# -*- coding: utf-8 -*- + +from enum import Enum + + +ENV_PREFIX = "BV_" +ENV_PREFIX_API = f"{ENV_PREFIX}CHALLENGE_API_" + + +class EnvEnum(str, Enum): + LOCAL = "LOCAL" + DEVELOPMENT = "DEVELOPMENT" + TEST = "TEST" + DEMO = "DEMO" + DOCS = "DOCS" + STAGING = "STAGING" + PRODUCTION = "PRODUCTION" + + +class WarnEnum(str, Enum): + ERROR = "ERROR" + ALWAYS = "ALWAYS" + DEBUG = "DEBUG" + IGNORE = "IGNORE" + + +class LanguageEnum(str, Enum): + en = "en" + ko = "ko" + mn = "mn" + + +class CurrencyEnum(str, Enum): + USD = "USD" + KRW = "KRW" + MNT = "MNT" + + +class HashAlgoEnum(str, Enum): + md5 = "md5" + sha1 = "sha1" + sha224 = "sha224" + sha256 = "sha256" + sha384 = "sha384" + sha512 = "sha512" + + +class HTTPSchemeEnum(str, Enum): + http = "http" + https = "https" -API_SLUG = "rest-bv-challenge" __all__ = [ "ENV_PREFIX", "ENV_PREFIX_API", - "API_SLUG", + "EnvEnum", + "WarnEnum", + "LanguageEnum", + "CurrencyEnum", + "HashAlgoEnum", + "HTTPSchemeEnum", ] diff --git a/src/bv_challenge/challenge/api/core/constants/_error_code.py b/src/bv_challenge/challenge/api/core/constants/_error_code.py index 76d2c9b..8a4c233 100644 --- a/src/bv_challenge/challenge/api/core/constants/_error_code.py +++ b/src/bv_challenge/challenge/api/core/constants/_error_code.py @@ -1,16 +1,20 @@ +# -*- coding: utf-8 -*- + from enum import Enum from http import HTTPStatus -from typing import Union, Any +from typing import Union, Optional, Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, constr class ErrorCodePM(BaseModel): - code: str = Field(..., min_length=3, max_length=36) - name: str = Field(..., min_length=3, max_length=64) + code: constr(strip_whitespace=True) = Field(..., min_length=3, max_length=36) # type: ignore + name: constr(strip_whitespace=True) = Field(..., min_length=3, max_length=64) # type: ignore status_code: int = Field(..., ge=100, le=599) - message: str = Field(..., min_length=1, max_length=256) - description: str | None = Field(default=None, max_length=1024) + message: constr(strip_whitespace=True) = Field(..., min_length=1, max_length=256) # type: ignore + description: Optional[constr(strip_whitespace=True)] = Field( # type: ignore + default=None, max_length=1024 + ) detail: Any = Field(default=None) diff --git a/src/bv_challenge/challenge/api/core/constants/_regex.py b/src/bv_challenge/challenge/api/core/constants/_regex.py index babab2c..3c4ad87 100644 --- a/src/bv_challenge/challenge/api/core/constants/_regex.py +++ b/src/bv_challenge/challenge/api/core/constants/_regex.py @@ -1,5 +1,51 @@ +# -*- coding: utf-8 -*- + # Valid characters: +ALPHANUM_REGEX = r"^[0-9a-zA-Z]+$" +ALPHANUM_SPACE_REGEX = r"^[0-9a-zA-Z ]+$" +ALPHANUM_HYPHEN_REGEX = r"^[0-9a-zA-Z_\-]+$" +ALPHANUM_HOST_REGEX = r"^[0-9a-zA-Z_\-.]+$" +ALPHANUM_EXTEND_REGEX = r"^[0-9a-zA-Z_\-. ]+$" +ALPHANUM_PATH_REGEX = r"^[0-9a-zA-Z_\-. \\\/]+$" + +REQUEST_ID_REGEX = ( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b|" + r"\b[0-9a-fA-F]{32}\b" +) + +ALPHANUM_CUSTOM_REGEX = r"^[0-9a-zA-Z_\-:+/=]+$" +REQUIREMENTS_REGEX = r"^[0-9a-zA-Z_\-.,\[\]!<>=~]+$" + +HTTP_METHOD_REGEX = r"^(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS|CONNECT|TRACE|\*)$" +ASYMMETRIC_ALGORITHM_REGEX = r"^(RS256|RS384|RS512)$" +JWT_ALGORITHM_REGEX = r"^(HS256|HS384|HS512|ES256|ES256K|ES384|ES512|RS256|RS384|RS512|PS256|PS384|PS512|EdDSA)$" + # Invalid characters: +SPECIAL_CHARS_REGEX = r"[&'\"<>]" +SPECIAL_CHARS_BASE_REGEX = r"[&'\"<>\\\/]" +SPECIAL_CHARS_LOW_REGEX = r"[&'\"<>\\\/`{}|]" +SPECIAL_CHARS_MEDIUM_REGEX = r"[&'\"<>\\\/`{}|()\[\]]" +SPECIAL_CHARS_HIGH_REGEX = r"[&'\"<>\\\/`{}|()\[\]!@#$%^*;:?]" +SPECIAL_CHARS_STRICT_REGEX = r"[&'\"<>\\\/`{}|()\[\]~!@#$%^*_=\-+;:,.?\t\n ]" -__all__ = [] +__all__ = [ + "ALPHANUM_REGEX", + "ALPHANUM_SPACE_REGEX", + "ALPHANUM_HYPHEN_REGEX", + "ALPHANUM_HOST_REGEX", + "ALPHANUM_EXTEND_REGEX", + "ALPHANUM_PATH_REGEX", + "ALPHANUM_CUSTOM_REGEX", + "REQUEST_ID_REGEX", + "REQUIREMENTS_REGEX", + "HTTP_METHOD_REGEX", + "ASYMMETRIC_ALGORITHM_REGEX", + "JWT_ALGORITHM_REGEX", + "SPECIAL_CHARS_REGEX", + "SPECIAL_CHARS_BASE_REGEX", + "SPECIAL_CHARS_LOW_REGEX", + "SPECIAL_CHARS_MEDIUM_REGEX", + "SPECIAL_CHARS_HIGH_REGEX", + "SPECIAL_CHARS_STRICT_REGEX", +] diff --git a/src/bv_challenge/challenge/api/core/dependencies/__init__.py b/src/bv_challenge/challenge/api/core/dependencies/__init__.py index e69de29..40a96af 100644 --- a/src/bv_challenge/challenge/api/core/dependencies/__init__.py +++ b/src/bv_challenge/challenge/api/core/dependencies/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/core/dependencies/auth.py b/src/bv_challenge/challenge/api/core/dependencies/auth.py index afe1068..a0e5de1 100644 --- a/src/bv_challenge/challenge/api/core/dependencies/auth.py +++ b/src/bv_challenge/challenge/api/core/dependencies/auth.py @@ -1,30 +1,30 @@ -from typing import Any +# -*- coding: utf-8 -*- + +from typing import Any, Dict, Optional, List from jwt import ExpiredSignatureError, InvalidTokenError from fastapi import Security, Depends, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer -from potato_util.constants import ALPHANUM_HOST_REGEX -from potato_util import validator -from potato_util.crypto import jwt as jwt_utils - -from api.core.constants import ErrorCodeEnum +from api.core.constants import ErrorCodeEnum, ALPHANUM_HOST_REGEX from api.config import config +from api.core.utils import validator +from api.helpers.crypto import jwt as jwt_helper from api.core.exceptions import BaseHTTPException + _http_bearer = HTTPBearer(auto_error=False) def auth_jwt( request: Request, - authorization: HTTPAuthorizationCredentials | None = Security(_http_bearer), -) -> dict[str, Any]: + authorization: Optional[HTTPAuthorizationCredentials] = Security(_http_bearer), +) -> Dict[str, Any]: """Dependency function to authenticate the access token (JWT) and get the payload. Args: request (Request , required): The FastAPI request object. - authorization (HTTPAuthorizationCredentials, required): 'Authorization: Bearer ' - header credentials. + authorization (HTTPAuthorizationCredentials, required): 'Authorization: Bearer ' header credentials. Raises: BaseHTTPException: If the access token is missing. @@ -32,7 +32,7 @@ def auth_jwt( BaseHTTPException: If the access token is invalid. Returns: - dict[str, Any]: The decoded access token payload. + Dict[str, Any]: The decoded access token payload. """ if not authorization: @@ -50,9 +50,9 @@ def auth_jwt( headers={"WWW-Authenticate": 'Bearer error="invalid_token"'}, ) - _payload: dict[str, Any] + _payload: Dict[str, Any] = None try: - _payload: dict[str, Any] = jwt_utils.decode( + _payload: Dict[str, Any] = jwt_helper.decode( token=_access_token, key=config.api.security.jwt.secret, algorithm=config.api.security.jwt.algorithm, @@ -74,17 +74,17 @@ def auth_jwt( return _payload -def get_user_id(payload: dict[str, Any] = Depends(auth_jwt)) -> str: +def get_user_id(payload: Dict[str, Any] = Depends(auth_jwt)) -> str: """Dependency function to get the user ID from the token payload. Args: - payload (dict[str, Any], required): The decoded access token payload. + payload (Dict[str, Any], required): The decoded access token payload. Returns: str: The user ID. """ - _user_id: str = payload.get("sub", "") + _user_id: str = payload.get("sub") return _user_id @@ -110,39 +110,36 @@ def __init__(self, allow_scope: str, allow_owner: bool = False): self.allow_owner = allow_owner def __call__( - self, request: Request, payload: dict[str, Any] = Depends(auth_jwt) - ) -> dict[str, Any]: + self, request: Request, payload: Dict[str, Any] = Depends(auth_jwt) + ) -> Dict[str, Any]: """Dependency function to check the scope permissions of the user. Args: request (Request , required): The FastAPI request object. - payload (dict[str, Any], required): The decoded access token (JWT) payload. + payload (Dict[str, Any], required): The decoded access token (JWT) payload. Raises: BaseHTTPException: If the user has insufficient scope permissions. Returns: - dict[str, Any]: The decoded access token payload. + Dict[str, Any]: The decoded access token payload. """ if self.allow_owner: - _auth_user_id: str = payload.get("sub", "") - _path_params: list[str] = list(request.path_params.values()) + _auth_user_id: str = payload.get("sub") + _path_params: List[str] = list(request.path_params.values()) if _path_params and (_path_params[0] == _auth_user_id): return payload - _token_all_scope: str = payload.get("scope", "") - _token_scope_list: list[str] = _token_all_scope.split(" ") + _token_all_scope: str = payload.get("scope") + _token_scope_list: List[str] = _token_all_scope.split(" ") if self.allow_scope not in _token_scope_list: raise BaseHTTPException( error_enum=ErrorCodeEnum.FORBIDDEN, message="You do not have enough scope permissions!", description="The request requires more scope permissions.", headers={ - "WWW-Authenticate": ( - 'Bearer error="insufficient_scope", ' - 'error_description="The request requires more scope permissions."' - ) + "WWW-Authenticate": 'Bearer error="insufficient_scope", error_description="The request requires more scope permissions."' }, ) diff --git a/src/bv_challenge/challenge/api/core/exceptions/__init__.py b/src/bv_challenge/challenge/api/core/exceptions/__init__.py index b722c5d..03e9716 100644 --- a/src/bv_challenge/challenge/api/core/exceptions/__init__.py +++ b/src/bv_challenge/challenge/api/core/exceptions/__init__.py @@ -1,3 +1,3 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * diff --git a/src/bv_challenge/challenge/api/core/exceptions/_base.py b/src/bv_challenge/challenge/api/core/exceptions/_base.py index f639d22..67918da 100644 --- a/src/bv_challenge/challenge/api/core/exceptions/_base.py +++ b/src/bv_challenge/challenge/api/core/exceptions/_base.py @@ -1,6 +1,8 @@ -from typing import Any, cast +# -*- coding: utf-8 -*- -from pydantic import validate_call +from typing import Any, Optional, Dict + +from pydantic import conint, constr, validate_call from fastapi import HTTPException from api.core.constants import ErrorCodeEnum @@ -17,36 +19,32 @@ class BaseHTTPException(HTTPException): def __init__( self, error_enum: ErrorCodeEnum, - status_code: int | None = None, - message: str | None = None, - content: Any = None, - description: str | None = None, + status_code: Optional[conint(ge=100, le=599)] = None, # type: ignore + message: Optional[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = None, + description: Optional[constr(strip_whitespace=True, max_length=1024)] = None, # type: ignore detail: Any = None, - headers: dict[str, str] | None = None, + headers: Optional[Dict[str, str]] = None, ): """Constructor method for BaseHTTPException class. Args: - error_enum (ErrorCodeEnum , required): Main error code enum. - status_code (int | None , optional): HTTP status code: [ge=100, le=599]. Defaults to None. - message (str | None , optional): Error message: [min_length=1, max_length=255]. - Defaults to None. - content (Any , optional): Any data content for response. Defaults to None. - description (str | None , optional): Error description: [max_length=511]. Defaults to None. - detail (Any , optional): Error detail. Defaults to None. - headers (dict[str, str] | None, optional): Headers. Defaults to None. + error_enum (ErrorCodeEnum , required): Main error code enum. + status_code (Optional[int] , optional): HTTP status code: [ge=100, le=599]. Defaults to None. + message (Optional[str] , optional): Error message: [min_length=1, max_length=255]. Defaults to None. + description (Optional[str] , optional): Error description: [max_length=511]. Defaults to None. + detail (Any , optional): Error detail. Defaults to None. + headers (Optional[Dict[str, str]], optional): Headers. Defaults to None. """ _error = error_enum.value.model_dump() if not status_code: - status_code = cast(int, _error.get("status_code", 500)) + status_code: int = _error.get("status_code") if not message: - message = _error.get("message", "An error occurred") - - if content: - self.content = content + message: str = _error.get("message") if description: _error["description"] = description diff --git a/src/bv_challenge/challenge/api/core/handlers/__init__.py b/src/bv_challenge/challenge/api/core/handlers/__init__.py index 0c4cade..1059366 100644 --- a/src/bv_challenge/challenge/api/core/handlers/__init__.py +++ b/src/bv_challenge/challenge/api/core/handlers/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._not_found import * from ._method_not_allowed import * diff --git a/src/bv_challenge/challenge/api/core/handlers/_http_exception.py b/src/bv_challenge/challenge/api/core/handlers/_http_exception.py index eb8b89e..8da773b 100644 --- a/src/bv_challenge/challenge/api/core/handlers/_http_exception.py +++ b/src/bv_challenge/challenge/api/core/handlers/_http_exception.py @@ -1,15 +1,16 @@ -from fastapi import HTTPException, Request +# -*- coding: utf-8 -*- + +from typing import Union -from potato_util.http import get_http_status +from fastapi import HTTPException, Request from api.core.constants import ErrorCodeEnum +from api.core import utils from api.core.responses import BaseResponse -# For HTTPException error: -async def http_exception_handler( - request: Request, exc: HTTPException | Exception -) -> BaseResponse: +## For HTTPException error: +async def http_exception_handler(request: Request, exc: HTTPException) -> BaseResponse: """HTTPException handler. Args: @@ -20,14 +21,10 @@ async def http_exception_handler( BaseResponse: Response object. """ - assert isinstance( - exc, HTTPException - ), f"`exc` argument type is invalid {type(exc)}, expected !" - _message: str - _error: dict | str | None = None + _error: Union[dict, str, None] = None - _http_status, _ = get_http_status(status_code=exc.status_code) + _http_status, _ = utils.get_http_status(status_code=exc.status_code) if isinstance(exc.detail, dict): _message = str(exc.detail.get("message", _http_status.phrase)) @@ -45,18 +42,12 @@ async def http_exception_handler( if _error_code_enum: _error = _error_code_enum.value.model_dump() - _content = None - if hasattr(exc, "content"): - _content = getattr(exc, "content") - - _headers = dict(exc.headers) if exc.headers else None return BaseResponse( request=request, - content=_content, status_code=exc.status_code, message=_message, error=_error, - headers=_headers, + headers=exc.headers, ) diff --git a/src/bv_challenge/challenge/api/core/handlers/_method_not_allowed.py b/src/bv_challenge/challenge/api/core/handlers/_method_not_allowed.py index 867e31c..80c4bd1 100644 --- a/src/bv_challenge/challenge/api/core/handlers/_method_not_allowed.py +++ b/src/bv_challenge/challenge/api/core/handlers/_method_not_allowed.py @@ -1,12 +1,14 @@ +# -*- coding: utf-8 -*- + from fastapi import HTTPException, Request from api.core.constants import ErrorCodeEnum from api.core.responses import BaseResponse -# For 405 status code: +## For 405 status code: async def method_not_allowed_handler( - request: Request, exc: HTTPException | Exception + request: Request, exc: HTTPException ) -> BaseResponse: """405 status code handler. @@ -19,7 +21,7 @@ async def method_not_allowed_handler( """ _error = ErrorCodeEnum.METHOD_NOT_ALLOWED.value.model_dump() - _message: str = _error.get("message", "Method Not Allowed") + _message: str = _error.get("message") return BaseResponse( request=request, status_code=405, message=_message, error=_error diff --git a/src/bv_challenge/challenge/api/core/handlers/_not_found.py b/src/bv_challenge/challenge/api/core/handlers/_not_found.py index 5158524..29205e9 100644 --- a/src/bv_challenge/challenge/api/core/handlers/_not_found.py +++ b/src/bv_challenge/challenge/api/core/handlers/_not_found.py @@ -1,13 +1,13 @@ +# -*- coding: utf-8 -*- + from fastapi import HTTPException, Request from api.core.constants import ErrorCodeEnum from api.core.responses import BaseResponse -# For 404 status code: -async def not_found_handler( - request: Request, exc: HTTPException | Exception -) -> BaseResponse: +## For 404 status code: +async def not_found_handler(request: Request, exc: HTTPException) -> BaseResponse: """404 status code handler. Args: @@ -18,11 +18,8 @@ async def not_found_handler( BaseResponse: Response object. """ - if not isinstance(exc, HTTPException): - exc = HTTPException(status_code=404) - _error = ErrorCodeEnum.NOT_FOUND.value.model_dump() - _message: str = _error.get("message", "Not Found") + _message: str = _error.get("message") if hasattr(exc, "detail") and isinstance(exc.detail, dict): _message = exc.detail.get("message", _message) diff --git a/src/bv_challenge/challenge/api/core/handlers/_server_error.py b/src/bv_challenge/challenge/api/core/handlers/_server_error.py index d350114..e7d8ba8 100644 --- a/src/bv_challenge/challenge/api/core/handlers/_server_error.py +++ b/src/bv_challenge/challenge/api/core/handlers/_server_error.py @@ -1,18 +1,17 @@ -from typing import Any +# -*- coding: utf-8 -*- from fastapi import Request -from beans_logging_fastapi import log_http_error +from beans_logging_fastapi import async_log_http_error from api.core.constants import ErrorCodeEnum from api.config import config from api.core.exceptions import PrimaryKeyError, UniqueKeyError from api.core.responses import BaseResponse +from api.logger import logger -# from api.logger import logger - -# For unhandled Exception or 500 internal server error: +## For unhandled Exception or 500 internal server error: async def server_error_handler(request: Request, exc: Exception) -> BaseResponse: """Error handler for any kind of unhandled Exception or 500 internal server error. @@ -30,18 +29,18 @@ async def server_error_handler(request: Request, exc: Exception) -> BaseResponse if isinstance(exc, UniqueKeyError): _error_enum = ErrorCodeEnum.DB_UQ_ERROR - # _request_id: str = request.state.request_id + _request_id: str = request.state.request_id _exc_str = str(exc) _status_code = _error_enum.value.status_code - _error: dict[str, Any] = _error_enum.value.model_dump() + _error = _error_enum.value.model_dump() _error["detail"] = _exc_str - _message: str = _error.get("message", "Internal Server Error") + _message: str = _error.get("message") - # logger.exception(f"[{_request_id}] {_error_enum.value.code} - {_exc_str}") - log_http_error( + logger.exception(f"[{_request_id}] {_error_enum.value.code} - {_exc_str}") + await async_log_http_error( request=request, status_code=_status_code, - msg_format_str=config.api.logger.http.std.err_msg_format_str, + msg_format=config.logger.extra.http_std_error_format, ) return BaseResponse( request=request, status_code=_status_code, message=_message, error=_error diff --git a/src/bv_challenge/challenge/api/core/handlers/_validation_error.py b/src/bv_challenge/challenge/api/core/handlers/_validation_error.py index a535a07..8735d3c 100644 --- a/src/bv_challenge/challenge/api/core/handlers/_validation_error.py +++ b/src/bv_challenge/challenge/api/core/handlers/_validation_error.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from fastapi import Request from fastapi.exceptions import RequestValidationError @@ -5,9 +7,9 @@ from api.core.responses import BaseResponse -# For RequestValidationError error: +## For RequestValidationError error: async def validation_error_handler( - request: Request, exc: RequestValidationError | Exception + request: Request, exc: RequestValidationError ) -> BaseResponse: """RequestValidationError handler for validation error. @@ -19,10 +21,6 @@ async def validation_error_handler( BaseResponse: Response object. """ - assert isinstance( - exc, RequestValidationError - ), f"`exc` argument type is invalid {type(exc)}, expected !" - _message = "Validation error!" _error = ErrorCodeEnum.UNPROCESSABLE_ENTITY.value.model_dump() _error["description"] = str(exc) diff --git a/src/bv_challenge/challenge/api/core/middlewares/__init__.py b/src/bv_challenge/challenge/api/core/middlewares/__init__.py index 1be8359..7271d04 100644 --- a/src/bv_challenge/challenge/api/core/middlewares/__init__.py +++ b/src/bv_challenge/challenge/api/core/middlewares/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._process_time import * from ._request_id import * diff --git a/src/bv_challenge/challenge/api/core/middlewares/_process_time.py b/src/bv_challenge/challenge/api/core/middlewares/_process_time.py index bbd08cf..f15fd8e 100644 --- a/src/bv_challenge/challenge/api/core/middlewares/_process_time.py +++ b/src/bv_challenge/challenge/api/core/middlewares/_process_time.py @@ -1,5 +1,7 @@ +# -*- coding: utf-8 -*- + import time -from collections.abc import Callable +from typing import Callable from starlette.middleware.base import BaseHTTPMiddleware from fastapi import Request, Response diff --git a/src/bv_challenge/challenge/api/core/middlewares/_request_id.py b/src/bv_challenge/challenge/api/core/middlewares/_request_id.py index 30abc14..573fb92 100644 --- a/src/bv_challenge/challenge/api/core/middlewares/_request_id.py +++ b/src/bv_challenge/challenge/api/core/middlewares/_request_id.py @@ -1,5 +1,7 @@ +# -*- coding: utf-8 -*- + from uuid import uuid4 -from collections.abc import Callable +from typing import Callable from starlette.middleware.base import BaseHTTPMiddleware from fastapi import Request, Response @@ -16,9 +18,9 @@ class RequestIdMiddleware(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next: Callable) -> Response: _request_id: str = uuid4().hex if "X-Request-ID" in request.headers: - _request_id: str = request.headers.get("X-Request-ID", _request_id) + _request_id: str = request.headers.get("X-Request-ID") elif "X-Correlation-ID" in request.headers: - _request_id: str = request.headers.get("X-Correlation-ID", _request_id) + _request_id: str = request.headers.get("X-Correlation-ID") request.state.request_id = _request_id response: Response = await call_next(request) diff --git a/src/bv_challenge/challenge/api/core/models/__init__.py b/src/bv_challenge/challenge/api/core/models/__init__.py index e69de29..40a96af 100644 --- a/src/bv_challenge/challenge/api/core/models/__init__.py +++ b/src/bv_challenge/challenge/api/core/models/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/core/responses/__init__.py b/src/bv_challenge/challenge/api/core/responses/__init__.py index b722c5d..03e9716 100644 --- a/src/bv_challenge/challenge/api/core/responses/__init__.py +++ b/src/bv_challenge/challenge/api/core/responses/__init__.py @@ -1,3 +1,3 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * diff --git a/src/bv_challenge/challenge/api/core/responses/_base.py b/src/bv_challenge/challenge/api/core/responses/_base.py index d05c268..3d8e9e0 100644 --- a/src/bv_challenge/challenge/api/core/responses/_base.py +++ b/src/bv_challenge/challenge/api/core/responses/_base.py @@ -1,17 +1,16 @@ +# -*- coding: utf-8 -*- + from http import HTTPStatus -from typing import Any +from typing import Any, Optional, Dict, Type -from pydantic import validate_call +from pydantic import validate_call, conint, constr from starlette.background import BackgroundTask from fastapi import Request from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse -from potato_util.http import get_http_status -from potato_util.http.fastapi import get_relative_url - -from api.__version__ import __version__ from api.config import config +from api.core import utils from api.core.schemas import BaseResPM @@ -27,44 +26,44 @@ class BaseResponse(JSONResponse): def __init__( self, content: Any = None, - status_code: int = 200, - headers: dict[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - request: Request | None = None, - message: str | None = None, - links: dict[str, Any] | None = None, - meta: dict[str, Any] | None = None, + status_code: Optional[conint(ge=100, le=599)] = 200, # type: ignore + headers: Optional[Dict[str, str]] = None, + media_type: Optional[constr(strip_whitespace=True)] = None, # type: ignore + background: Optional[BackgroundTask] = None, + request: Optional[Request] = None, + message: Optional[ + constr(strip_whitespace=True, min_length=1, max_length=256) # type: ignore + ] = None, + links: Optional[Dict[str, Any]] = None, + meta: Optional[Dict[str, Any]] = None, error: Any = None, - response_schema: type[BaseResPM] = BaseResPM, + response_schema: Optional[Type[BaseResPM]] = BaseResPM, ) -> None: """Constructor method for BaseResponse class. This will prepare the most response data and pass it to `JSONResponse` parent class constructor. Args: - content (Any , optional): Main data content for response. Defaults to None. - status_code (int | None , optional): HTTP status code: [100 <= status_code <= 599]. - Defaults to 200. - headers (dict[str, str] | None , optional): HTTP headers. Defaults to None. - media_type (str | None , optional): Media type for 'Content-Type' header. Defaults to None. - background (BackgroundTask | None , optional): Background task. Defaults to None. - request (Request | None , optional): Request object from FastAPI. Defaults to None. - message (str | None , optional): Message for response: [1 <= len(message) <= 256]. - Defaults to None. - links (dict[str, Any] | None , optional): Links for response. Defaults to None. - meta (dict[str, Any] | None , optional): Meta data for response. Defaults to None. - error (Any , optional): Error data for response. Defaults to None. - response_schema (type[BaseResPM] | None, optional): Response schema type. Defaults to `Type[BaseResPM]`. + content (Any , optional): Main data content for response. Defaults to None. + status_code (Optional[int] , optional): HTTP status code: [100 <= status_code <= 599]. Defaults to 200. + headers (Optional[Dict[str, str]] , optional): HTTP headers. Defaults to None. + media_type (Optional[str] , optional): Media type for 'Content-Type' header. Defaults to None. + background (Optional[BackgroundTask] , optional): Background task. Defaults to None. + request (Optional[Request] , optional): Request object from FastAPI. Defaults to None. + message (Optional[str] , optional): Message for response: [1 <= len(message) <= 256]. Defaults to None. + links (Optional[Dict[str, Any]] , optional): Links for response. Defaults to None. + meta (Optional[Dict[str, Any]] , optional): Meta data for response. Defaults to None. + error (Any , optional): Error data for response. Defaults to None. + response_schema (Optional[Type[BaseResPM]], optional): Response schema type. Defaults to `Type[BaseResPM]`. """ _http_status: HTTPStatus - _http_status, _ = get_http_status(status_code=status_code) + _http_status, _ = utils.get_http_status(status_code=status_code) if not message: if error and isinstance(error, dict) and ("message" in error): message = str(error["message"]) else: - message = _http_status.phrase + message: str = _http_status.phrase if not links: links = {} @@ -78,19 +77,20 @@ def __init__( if request: _request_id: str = request.state.request_id - links["self"] = f"{get_relative_url(request)}" + links["self"] = f"{utils.get_relative_url(request)}" + meta["request_id"] = _request_id meta["method"] = request.method meta["base_url"] = str(request.base_url)[:-1] if "X-Request-Id" not in headers: headers["X-Request-Id"] = _request_id - headers["X-API-Version"] = config.api.version - headers["X-System-Version"] = __version__ + meta["api_version"] = config.api.version + meta["version"] = config.version if error and isinstance(error, dict): if ("code" in error) and ("X-Error-Code" not in headers): - headers["X-Error-Code"] = error.get("code", f"{status_code}_00000") + headers["X-Error-Code"] = error.get("code") if (not config.debug) and (500 <= status_code) and ("detail" in error): error["detail"] = None @@ -115,7 +115,7 @@ def __init__( headers["Retry-After"] = "1800" _response_pm = response_schema( - message=message, data=content, links=links, meta=meta, error=error # type: ignore + message=message, data=content, links=links, meta=meta, error=error ) _content = jsonable_encoder(obj=_response_pm, by_alias=True) diff --git a/src/bv_challenge/challenge/api/core/routers/__init__.py b/src/bv_challenge/challenge/api/core/routers/__init__.py index e69de29..40a96af 100644 --- a/src/bv_challenge/challenge/api/core/routers/__init__.py +++ b/src/bv_challenge/challenge/api/core/routers/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/core/routers/default.py b/src/bv_challenge/challenge/api/core/routers/default.py index a38c616..dee3054 100644 --- a/src/bv_challenge/challenge/api/core/routers/default.py +++ b/src/bv_challenge/challenge/api/core/routers/default.py @@ -1,8 +1,11 @@ +# -*- coding: utf-8 -*- + from fastapi import APIRouter from fastapi.responses import RedirectResponse from api.config import config + router = APIRouter(tags=["Default"]) @@ -17,6 +20,7 @@ async def get_root(): if config.api.docs.enabled: + if config.api.docs.openapi_url: @router.get( diff --git a/src/bv_challenge/challenge/api/core/routers/utils.py b/src/bv_challenge/challenge/api/core/routers/utils.py index 06a112b..0d0cf44 100644 --- a/src/bv_challenge/challenge/api/core/routers/utils.py +++ b/src/bv_challenge/challenge/api/core/routers/utils.py @@ -1,8 +1,12 @@ -from fastapi import APIRouter, Request +# -*- coding: utf-8 -*- -from api.core.schemas import BaseResPM +from fastapi import APIRouter, Request, Response +from fastapi.responses import JSONResponse + +from api.core.schemas import BaseResPM, HealthResPM from api.core.responses import BaseResponse + router = APIRouter(tags=["Utils"]) @@ -32,22 +36,15 @@ async def get_ping(request: Request): "/health", summary="Health", description="Check health of all related backend services.", - response_model=BaseResPM, + response_class=JSONResponse, + response_model=HealthResPM, ) -async def get_health(request: Request): - _message = "Everything is OK." - _data = {"api": {"message": "API is up.", "is_alive": True}} +async def get_health(response: Response): + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" - return BaseResponse( - request=request, - content=_data, - message=_message, - headers={ - "Cache-Control": "no-cache, no-store, must-revalidate", - "Pragma": "no-cache", - "Expires": "0", - }, - ) + return {"status": "healthy"} __all__ = ["router"] diff --git a/src/bv_challenge/challenge/api/core/schemas/__init__.py b/src/bv_challenge/challenge/api/core/schemas/__init__.py index 6bd70cf..8a20231 100644 --- a/src/bv_challenge/challenge/api/core/schemas/__init__.py +++ b/src/bv_challenge/challenge/api/core/schemas/__init__.py @@ -1,4 +1,4 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * from ._responses import * diff --git a/src/bv_challenge/challenge/api/core/schemas/_base.py b/src/bv_challenge/challenge/api/core/schemas/_base.py index 79a9c4b..c026943 100644 --- a/src/bv_challenge/challenge/api/core/schemas/_base.py +++ b/src/bv_challenge/challenge/api/core/schemas/_base.py @@ -1,13 +1,14 @@ +# -*- coding: utf-8 -*- + from datetime import datetime from pydantic import BaseModel, Field, constr, ConfigDict -from potato_util.dt import now_utc_dt -from potato_util.generator import gen_unique_id +from api.core import utils class BasePM(BaseModel): - # model_config = ConfigDict(json_encoders={datetime: dt_to_iso}) + # model_config = ConfigDict(json_encoders={datetime: utils.datetime_to_iso}) pass @@ -19,7 +20,7 @@ class ExtraBasePM(BaseModel): class IdPM(BasePM): id: constr(strip_whitespace=True) = Field( # type: ignore - default_factory=gen_unique_id, + default_factory=utils.gen_unique_id, min_length=8, max_length=64, title="ID", @@ -30,16 +31,16 @@ class IdPM(BasePM): class TimestampPM(BasePM): updated_at: datetime = Field( - default_factory=now_utc_dt, + default_factory=utils.now_utc_dt, title="Updated datetime", description="Last updated datetime of the resource.", - examples=["2026-01-01T00:00:00+00:00"], + examples=["2024-12-01T00:00:00+00:00"], ) created_at: datetime = Field( - default_factory=now_utc_dt, + default_factory=utils.now_utc_dt, title="Created datetime", description="Created datetime of the resource.", - examples=["2026-01-01T00:00:00+00:00"], + examples=["2024-12-01T00:00:00+00:00"], ) diff --git a/src/bv_challenge/challenge/api/core/schemas/_error_responses.py b/src/bv_challenge/challenge/api/core/schemas/_error_responses.py index 488c34b..e13d6b8 100644 --- a/src/bv_challenge/challenge/api/core/schemas/_error_responses.py +++ b/src/bv_challenge/challenge/api/core/schemas/_error_responses.py @@ -1,4 +1,6 @@ -from typing import Any +# -*- coding: utf-8 -*- + +from typing import Any, Union from pydantic import Field @@ -14,13 +16,13 @@ class BadBaseResPM(BaseResPM): description="Response message about the current request.", examples=["Bad Request!"], ) - data: Any | dict | list = Field( + data: Union[Any, dict, list] = Field( default=None, title="Data", description="Resource data or any response related data.", examples=[None], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -43,7 +45,7 @@ class UnauthorizedBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Unauthorized!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -66,7 +68,7 @@ class ForbiddenBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Forbidden!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -89,7 +91,7 @@ class NotFoundBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Not Found!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -112,7 +114,7 @@ class MethodNotBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Method Not Allowed!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -135,7 +137,7 @@ class ConflictBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Conflict!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -158,7 +160,7 @@ class InvalidBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Validation error!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -188,7 +190,7 @@ class ErrorBaseResPM(BadBaseResPM): description="Response message about the current request.", examples=["Internal Server Error!"], ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", diff --git a/src/bv_challenge/challenge/api/core/schemas/_responses.py b/src/bv_challenge/challenge/api/core/schemas/_responses.py index 8c41aa8..fe9cfdd 100644 --- a/src/bv_challenge/challenge/api/core/schemas/_responses.py +++ b/src/bv_challenge/challenge/api/core/schemas/_responses.py @@ -1,18 +1,29 @@ -from typing import Any +# -*- coding: utf-8 -*- -from pydantic import Field +from enum import Enum +from typing import Any, Union, Optional -from potato_util.constants import HTTPMethodEnum +from pydantic import Field, constr from api.config import config - from ._base import ExtraBasePM, BasePM +class MethodEnum(str, Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + PATCH = "PATCH" + DELETE = "DELETE" + HEAD = "HEAD" + OPTIONS = "OPTIONS" + CONNECT = "CONNECT" + TRACE = "TRACE" + + class LinksResPM(ExtraBasePM): - self_link: str | None = Field( + self_link: Optional[constr(strip_whitespace=True, max_length=2048)] = Field( # type: ignore default=None, - max_length=2048, alias="self", title="Self link", description="Link to the current resource.", @@ -21,33 +32,29 @@ class LinksResPM(ExtraBasePM): class PageLinksResPM(LinksResPM): - first_link: str | None = Field( + first_link: Optional[constr(strip_whitespace=True, max_length=2048)] = Field( # type: ignore default=None, - max_length=2048, alias="first", title="First link", description="Link to the first page of the resource.", examples=[f"{config.api.prefix}/resources/?skip=0&limit=100"], ) - prev_link: str | None = Field( + prev_link: Optional[constr(strip_whitespace=True, max_length=2048)] = Field( # type: ignore default=None, - max_length=2048, alias="prev", title="Previous link", description="Link to the previous page of the resource.", examples=[f"{config.api.prefix}/resources/?skip=100&limit=100"], ) - next_link: str | None = Field( + next_link: Optional[constr(strip_whitespace=True, max_length=2048)] = Field( # type: ignore default=None, - max_length=2048, alias="next", title="Next link", description="Link to the next page of the resource.", examples=[f"{config.api.prefix}/resources/?skip=300&limit=100"], ) - last_link: str | None = Field( + last_link: Optional[constr(strip_whitespace=True, max_length=2048)] = Field( # type: ignore default=None, - max_length=2048, alias="last", title="Last link", description="Link to the last page of the resource.", @@ -56,24 +63,48 @@ class PageLinksResPM(LinksResPM): class MetaResPM(ExtraBasePM): - base_url: str | None = Field( + request_id: Optional[ + constr(strip_whitespace=True, min_length=8, max_length=64) # type: ignore + ] = Field( + default=None, + title="Request ID", + description="Current request ID.", + examples=["211203afa2844d55b1c9d38b9f8a7063"], + ) + base_url: Optional[ + constr(strip_whitespace=True, min_length=2, max_length=256) # type: ignore + ] = Field( default=None, - min_length=2, - max_length=256, title="Base URL", description="Current request base URL.", examples=["https://api.example.com"], ) - method: HTTPMethodEnum | None = Field( + method: Optional[MethodEnum] = Field( default=None, title="Method", description="Current request method.", examples=["GET"], ) + api_version: constr(strip_whitespace=True) = Field( # type: ignore + default=config.api.version, + min_length=1, + max_length=16, + title="API version", + description="Current API version.", + examples=[config.api.version], + ) + version: constr(strip_whitespace=True) = Field( # type: ignore + default=config.version, + min_length=5, + max_length=32, + title="Version", + description="Current system version.", + examples=[config.version], + ) class ErrorResPM(BasePM): - code: str = Field( + code: constr(strip_whitespace=True) = Field( # type: ignore ..., min_length=3, max_length=36, @@ -81,14 +112,14 @@ class ErrorResPM(BasePM): description="Code that represents the error.", examples=["400_00000"], ) - description: str | None = Field( + description: Optional[constr(strip_whitespace=True)] = Field( # type: ignore default=None, max_length=1024, title="Error description", description="Description of the error.", examples=["Bad request syntax or unsupported method."], ) - detail: Any | dict | list = Field( + detail: Union[Any, dict, list] = Field( default=None, title="Error detail", description="Detail of the error.", @@ -112,7 +143,7 @@ class BaseResPM(BasePM): description="Response message about the current request.", examples=["Successfully processed the request."], ) - data: Any | dict | list = Field( + data: Union[Any, dict, list] = Field( default=None, title="Data", description="Resource data or any data related to response.", @@ -128,7 +159,7 @@ class BaseResPM(BasePM): title="Meta", description="Meta information about the current request.", ) - error: ErrorResPM | Any = Field( + error: Union[ErrorResPM, Any] = Field( default=None, title="Error", description="Error information about the current request.", @@ -136,10 +167,22 @@ class BaseResPM(BasePM): ) +class HealthResPM(BasePM): + status: str = Field( + default="healthy", + min_length=2, + max_length=32, + title="Status", + description="Health status of the service.", + examples=["healthy"], + ) + + __all__ = [ "LinksResPM", "PageLinksResPM", "MetaResPM", "ErrorResPM", "BaseResPM", + "HealthResPM", ] diff --git a/src/bv_challenge/challenge/api/core/services/__init__.py b/src/bv_challenge/challenge/api/core/services/__init__.py index e69de29..40a96af 100644 --- a/src/bv_challenge/challenge/api/core/services/__init__.py +++ b/src/bv_challenge/challenge/api/core/services/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/core/utils/__init__.py b/src/bv_challenge/challenge/api/core/utils/__init__.py index b722c5d..3b1b591 100644 --- a/src/bv_challenge/challenge/api/core/utils/__init__.py +++ b/src/bv_challenge/challenge/api/core/utils/__init__.py @@ -1,3 +1,9 @@ -# flake8: noqa +# -*- coding: utf-8 -*- from ._base import * +from ._secure import * +from ._http import * +from ._dt import * +from ._io import * +from . import _validator as validator +from . import _sanitizer as sanitizer diff --git a/src/bv_challenge/challenge/api/core/utils/_base.py b/src/bv_challenge/challenge/api/core/utils/_base.py index c39d6b6..f3fa003 100644 --- a/src/bv_challenge/challenge/api/core/utils/_base.py +++ b/src/bv_challenge/challenge/api/core/utils/_base.py @@ -1,36 +1,117 @@ -import sys -from functools import lru_cache - -BINARY_MODULES = [ - "uvicorn", - "gunicorn", - "fastapi", - "pytest", - "unittest", - "alembic", -] +# -*- coding: utf-8 -*- + +import re +import copy + +from pydantic import validate_call + +from beans_logging import logger -@lru_cache -def is_running_bin() -> bool: - """Checks if the application is running as a binary environment module (e.g., via uvicorn, fastapi, gunicorn, etc.) - by inspecting the command-line arguments. +@validate_call +def deep_merge(dict1: dict, dict2: dict) -> dict: + """Return a new dictionary that's the result of a deep merge of two dictionaries. + If there are conflicts, values from `dict2` will overwrite those in `dict1`. + + Args: + dict1 (dict, required): The base dictionary that will be merged. + dict2 (dict, required): The dictionary to merge into `dict1`. Returns: - bool: True if running as a binary environment module, False otherwise. + dict: The merged dictionary. """ - for _binary_module in BINARY_MODULES: + _merged = copy.deepcopy(dict1) + for _key, _val in dict2.items(): if ( - sys.argv[0].endswith(_binary_module) - or sys.argv[0].endswith(f"{_binary_module}.exe") - or sys.argv[0].endswith(f"{_binary_module}/__main__.py") + _key in _merged + and isinstance(_merged[_key], dict) + and isinstance(_val, dict) ): - return True + _merged[_key] = deep_merge(_merged[_key], _val) + else: + _merged[_key] = copy.deepcopy(_val) + + return _merged + + +@validate_call +def camel_to_snake(val: str) -> str: + """Convert CamelCase to snake_case. + + Args: + val (str): CamelCase string to convert. + + Returns: + str: Converted snake_case string. + """ + + val = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", val) + val = re.sub("([a-z0-9])([A-Z])", r"\1_\2", val).lower() + return val + + +@validate_call +def clean_obj_dict(obj_dict: dict, cls_name: str) -> dict: + """Clean class name from object.__dict__ for str(object). + + Args: + obj_dict (dict, required): Object dictionary by object.__dict__. + cls_name (str , required): Class name by cls.__name__. + + Returns: + dict: Clean object dictionary. + """ + + try: + if not obj_dict: + raise ValueError("'obj_dict' argument value is empty!") + + if not cls_name: + raise ValueError("'cls_name' argument value is empty!") + except ValueError as err: + logger.error(err) + raise + + _self_dict = obj_dict.copy() + for _key in _self_dict.copy(): + _class_prefix = f"_{cls_name}__" + if _key.startswith(_class_prefix): + _new_key = _key.replace(_class_prefix, "") + _self_dict[_new_key] = _self_dict.pop(_key) + return _self_dict + + +@validate_call(config={"arbitrary_types_allowed": True}) +def obj_to_repr(obj: object) -> str: + """Modifying object default repr() to custom info. + + Args: + obj (object, required): Any python object. + + Returns: + str: String for repr() method. + """ + + try: + if not obj: + raise ValueError("'obj' argument value is empty!") + except ValueError as err: + logger.error(err) + raise - return False + _self_repr = ( + f"<{obj.__class__.__module__}.{obj.__class__.__name__} object at {hex(id(obj))}: " + + "{" + + f"{str(dir(obj)).replace('[', '').replace(']', '')}" + + "}>" + ) + return _self_repr __all__ = [ - "is_running_bin", + "deep_merge", + "camel_to_snake", + "clean_obj_dict", + "obj_to_repr", ] diff --git a/src/bv_challenge/challenge/api/core/utils/_dt.py b/src/bv_challenge/challenge/api/core/utils/_dt.py new file mode 100644 index 0000000..88a2648 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_dt.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- + +import time +from enum import Enum +from typing import Union, Optional +from zoneinfo import ZoneInfo +from datetime import datetime, timezone, tzinfo, timedelta + +from pydantic import validate_call, constr, conint +from beans_logging import logger + +from api.core.constants import WarnEnum + + +class TSUnitEnum(str, Enum): + SECONDS = "SECONDS" + MILLISECONDS = "MILLISECONDS" + MICROSECONDS = "MICROSECONDS" + NANOSECONDS = "NANOSECONDS" + + +@validate_call(config={"arbitrary_types_allowed": True}) +def add_tzinfo(dt: datetime, tz: Union[ZoneInfo, tzinfo, str]) -> datetime: + """Add or replace timezone info to datetime object. + + Args: + dt (datetime , required): Datetime object. + tz (Union[ZoneInfo, tzinfo, str], required): Timezone info. + + Returns: + datetime: Datetime object with timezone info. + """ + + if isinstance(tz, str): + tz = ZoneInfo(tz) + + dt = dt.replace(tzinfo=tz) + return dt + + +@validate_call +def datetime_to_iso( + dt: datetime, + sep: constr(max_length=8) = "T", # type: ignore + warn_mode: WarnEnum = WarnEnum.IGNORE, +) -> str: + """Convert datetime object to ISO 8601 format. + + Args: + dt (datetime, required): Datetime object. + sep (str , optional): Separator between date and time. Defaults to "T". + warn_mode (WarnEnum, optional): Warning mode. Defaults to WarnEnum.IGNORE. + + Raises: + ValueError: If `dt` argument doesn't have any timezone info and `warn_mode` is set to WarnEnum.ERROR. + + Returns: + str: Datetime string in ISO 8601 format. + """ + + if not dt.tzinfo: + _message = "Not found any timezone info in `dt` argument, assuming it's UTC timezone..." + if warn_mode == WarnEnum.ALWAYS: + logger.warning(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + elif warn_mode == WarnEnum.ERROR: + _message = "Not found any timezone info in `dt` argument!" + logger.error(_message) + raise ValueError(_message) + + dt = add_tzinfo(dt=dt, tz="UTC") + + _dt_str = dt.isoformat(sep=sep, timespec="milliseconds") + return _dt_str + + +@validate_call(config={"arbitrary_types_allowed": True}) +def convert_tz( + dt: datetime, + tz: Union[ZoneInfo, tzinfo, str], + warn_mode: WarnEnum = WarnEnum.ALWAYS, +) -> datetime: + """Convert datetime object to another timezone. + + Args: + dt (datetime , required): Datetime object to convert. + tz (Union[ZoneInfo, tzinfo, str], required): Timezone info to convert. + warn_mode (WarnEnum , optional): Warning mode. Defaults to WarnEnum.ALWAYS. + + Raises: + ValueError: If `dt` argument doesn't have any timezone info and `warn_mode` is set to WarnEnum.ERROR. + + Returns: + datetime: Datetime object which has been converted to another timezone. + """ + + if not dt.tzinfo: + _message = "Not found any timezone info in `dt` argument, assuming it's UTC timezone..." + if warn_mode == WarnEnum.ALWAYS: + logger.warning(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + elif warn_mode == WarnEnum.ERROR: + _message = "Not found any timezone info in `dt` argument!" + logger.error(_message) + raise ValueError(_message) + + dt = add_tzinfo(dt=dt, tz="UTC") + + if isinstance(tz, str): + tz = ZoneInfo(tz) + + dt = dt.astimezone(tz=tz) + return dt + + +def now_utc_dt() -> datetime: + """Get current datetime in UTC timezone with tzinfo. + + Returns: + datetime: Current datetime in UTC timezone with tzinfo. + """ + + _utc_dt = datetime.now(tz=timezone.utc) + return _utc_dt + + +def now_local_dt() -> datetime: + """Get current datetime in local timezone with tzinfo. + + Returns: + datetime: Current datetime in local timezone with tzinfo. + """ + + _local_dt = datetime.now().astimezone() + return _local_dt + + +@validate_call(config={"arbitrary_types_allowed": True}) +def now_dt(tz: Union[ZoneInfo, tzinfo, str]) -> datetime: + """Get current datetime in specified timezone with tzinfo. + + Args: + tz (Union[ZoneInfo, tzinfo, str], required): Timezone info. + + Returns: + datetime: Current datetime in specified timezone with tzinfo. + """ + + _dt = now_utc_dt() + _dt = convert_tz(dt=_dt, tz=tz) + return _dt + + +@validate_call +def now_ts(unit: TSUnitEnum = TSUnitEnum.SECONDS) -> int: + """Get current timestamp in UTC timezone. + + Args: + unit (TSUnitEnum, optional): Type of timestamp unit. Defaults to `TSUnitEnum.SECONDS`. + + Returns: + int: Current timestamp. + """ + + _now_ts: int = None + if unit == TSUnitEnum.SECONDS: + _now_ts = int(time.time()) + elif unit == TSUnitEnum.MILLISECONDS: + _now_ts = int(_now_ts * 1000) + elif unit == TSUnitEnum.MICROSECONDS: + _now_ts = int(time.time_ns() / 1000) + elif unit == TSUnitEnum.NANOSECONDS: + _now_ts = int(time.time_ns()) + + return _now_ts + + +@validate_call +def convert_ts(dt: datetime, unit: TSUnitEnum = TSUnitEnum.SECONDS) -> int: + """Convert datetime to timestamp. + + Args: + dt (datetime , required): Datetime object to convert. + unit (TSUnitEnum, optional): Type of timestamp unit. Defaults to `TSUnitEnum.SECONDS`. + + Returns: + int: Converted timestamp. + """ + + _ts: int = None + if unit == TSUnitEnum.SECONDS: + _ts = int(dt.timestamp()) + elif unit == TSUnitEnum.MILLISECONDS: + _ts = int(dt.timestamp() * 1000) + elif unit == TSUnitEnum.MICROSECONDS: + _ts = int(dt.timestamp() * 1000000) + elif unit == TSUnitEnum.NANOSECONDS: + _ts = int(dt.timestamp() * 1000000000) + + return _ts + + +@validate_call(config={"arbitrary_types_allowed": True}) +def calc_future_dt( + delta: Union[timedelta, conint(ge=1)], # type: ignore + dt: Optional[datetime] = None, + tz: Union[ZoneInfo, tzinfo, str, None] = None, +) -> datetime: + """Calculate future datetime by adding delta time to current or specified datetime. + + Args: + delta (Union[timedelta, int] , required): Delta time to add to current or specified datetime. + dt (Optional[datetime] , optional): Datetime before adding delta time. Defaults to None. + tz (Union[ZoneInfo, tzinfo, str, None], optional): Timezone info. Defaults to None. + + Returns: + datetime: Calculated future datetime. + """ + + if not dt: + dt = now_utc_dt() + + if tz: + dt = convert_tz(dt=dt, tz=tz) + + if isinstance(delta, int): + delta = timedelta(seconds=delta) + + _future_dt = dt + delta + return _future_dt + + +__all__ = [ + "add_tzinfo", + "datetime_to_iso", + "convert_tz", + "now_utc_dt", + "now_local_dt", + "now_dt", + "now_ts", + "convert_ts", + "calc_future_dt", +] diff --git a/src/bv_challenge/challenge/api/core/utils/_http.py b/src/bv_challenge/challenge/api/core/utils/_http.py new file mode 100644 index 0000000..cf52bd3 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_http.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +from typing import Tuple, Union +from urllib import request +from http import HTTPStatus +from http.client import HTTPResponse + +import aiohttp +from pydantic import validate_call, conint, AnyHttpUrl +from starlette.datastructures import URL +from fastapi import Request + + +@validate_call +def get_http_status(status_code: int) -> Tuple[HTTPStatus, bool]: + """Get HTTP status code enum from integer value. + + Args: + status_code (int, required): Status code for HTTP response: [100 <= status_code <= 599]. + + Raises: + ValueError: If status code is not in range [100 <= status_code <= 599]. + + Returns: + Tuple[HTTPStatus, bool]: Tuple of HTTP status code enum and boolean value if status code is known. + """ + + _http_status: HTTPStatus + _is_known_status = False + try: + _http_status = HTTPStatus(status_code) + _is_known_status = True + except ValueError: + if (100 <= status_code) and (status_code < 200): + status_code = 100 + elif (200 <= status_code) and (status_code < 300): + status_code = 200 + elif (300 <= status_code) and (status_code < 400): + status_code = 304 + elif (400 <= status_code) and (status_code < 500): + status_code = 400 + elif (500 <= status_code) and (status_code < 600): + status_code = 500 + else: + raise ValueError(f"Invalid HTTP status code: '{status_code}'!") + + _http_status = HTTPStatus(status_code) + + return (_http_status, _is_known_status) + + +@validate_call(config={"arbitrary_types_allowed": True}) +def get_relative_url(val: Union[Request, URL]) -> str: + """Get relative url only path with query params from request object or URL object. + + Args: + val (Union[Request, URL]): Request object or URL object to extract relative url. + + Returns: + str: Relative url only path with query params. + """ + + if isinstance(val, Request): + val: URL = val.url + + _relative_url = str(val).replace(f"{val.scheme}://{val.netloc}", "") + return _relative_url + + +@validate_call +async def async_is_connectable( + url: AnyHttpUrl = "https://www.google.com", + timeout: conint(ge=1) = 3, # type: ignore + check_status: bool = False, +) -> bool: + """Check if the url is connectable. + + Args: + url (AnyHttpUrl, optional): URL to check. Defaults to 'https://www.google.com'. + timeout (int , optional): Timeout in seconds. Defaults to 3. + check_status (bool , optional): Check HTTP status code (200). Defaults to False. + + Returns: + bool: True if connectable, False otherwise. + """ + + try: + async with aiohttp.ClientSession() as _session: + async with _session.get(url, timeout=timeout) as _response: + if check_status: + return _response.status == 200 + return True + except: + return False + + +@validate_call +def is_connectable( + url: AnyHttpUrl = "https://www.google.com", + timeout: conint(ge=1) = 3, # type: ignore + check_status: bool = False, +) -> bool: + """Check if the url is connectable. + + Args: + url (AnyHttpUrl, optional): URL to check. Defaults to 'https://www.google.com'. + timeout (int , optional): Timeout in seconds. Defaults to 3. + check_status (bool , optional): Check HTTP status code (200). Defaults to False. + + Returns: + bool: True if connectable, False otherwise. + """ + + try: + _response: HTTPResponse = request.urlopen(url, timeout=timeout) + if check_status: + return _response.getcode() == 200 + return True + except: + return False + + +__all__ = [ + "get_http_status", + "get_relative_url", + "async_is_connectable", + "is_connectable", +] diff --git a/src/bv_challenge/challenge/api/core/utils/_io.py b/src/bv_challenge/challenge/api/core/utils/_io.py new file mode 100644 index 0000000..06f2ddc --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_io.py @@ -0,0 +1,461 @@ +# -*- coding: utf-8 -*- + +import os +import errno +import shutil +import hashlib +from typing import List + +import aioshutil +import aiofiles.os +from pydantic import validate_call, conint, constr +from beans_logging import logger + +from api.core.constants import WarnEnum, HashAlgoEnum + + +_path_max_length = 1024 + + +## Async: +@validate_call +async def async_create_dir( + create_dir: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Asynchronous create directory if `create_dir` doesn't exist. + + Args: + create_dir (str, required): Create directory path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and directory already exists. + OSError: If failed to create directory. + """ + + if not await aiofiles.os.path.isdir(create_dir): + try: + _message = f"Creating '{create_dir}' directory..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + await aiofiles.os.makedirs(create_dir) + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{create_dir}' directory already exists!") + else: + logger.error(f"Failed to create '{create_dir}' directory!") + raise + + _message = f"Successfully created '{create_dir}' directory." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.EEXIST, f"'{create_dir}' directory already exists!") + + return + + +@validate_call +async def async_remove_dir( + remove_dir: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Asynchronous remove directory if `remove_dir` exists. + + Args: + remove_dir (str, required): Remove directory path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and directory doesn't exist. + OSError: If failed to remove directory. + """ + + if await aiofiles.os.path.isdir(remove_dir): + try: + _message = f"Removing '{remove_dir}' directory..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + await aioshutil.rmtree(remove_dir) + except OSError as err: + if (err.errno == errno.ENOENT) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{remove_dir}' directory doesn't exist!") + else: + logger.error(f"Failed to remove '{remove_dir}' directory!") + raise + + _message = f"Successfully removed '{remove_dir}' directory." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, f"'{create_dir}' directory doesn't exist!") + + return + + +@validate_call +async def async_remove_dirs( + remove_dirs: List[constr(strip_whitespace=True, min_length=1, max_length=_path_max_length)], # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Asynchronous remove directories if `remove_dirs` exists. + + Args: + remove_dirs (List[str], required): Remove directories paths as list. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + """ + + for _remove_dir in remove_dirs: + await async_remove_dir(remove_dir=_remove_dir, warn_mode=warn_mode) + + return + + +@validate_call +async def async_remove_file( + file_path: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Asynchronous remove file if `file_path` exists. + + Args: + file_path (str, required): Remove file path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and file doesn't exist. + OSError: If failed to remove file. + """ + + if await aiofiles.os.path.isfile(file_path): + try: + _message = f"Removing '{file_path}' file..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + await aiofiles.os.remove(file_path) + except OSError as err: + if (err.errno == errno.ENOENT) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{file_path}' file doesn't exist!") + else: + logger.error(f"Failed to remove '{file_path}' file!") + raise + + _message = f"Successfully removed '{file_path}' file." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, f"'{file_path}' file doesn't exist!") + + return + + +@validate_call +async def async_remove_files( + file_paths: List[constr(strip_whitespace=True, min_length=1, max_length=_path_max_length)], # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Asynchronous remove files if `file_paths` exists. + + Args: + file_paths (List[str], required): Remove file paths as list. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + """ + + for _file_path in file_paths: + await async_remove_file(file_path=_file_path, warn_mode=warn_mode) + + return + + +@validate_call +async def async_get_file_checksum( + file_path: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + hash_method: HashAlgoEnum = HashAlgoEnum.md5, + chunk_size: conint(ge=10) = 4096, # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> str: + """Asynchronous get file checksum. + + Args: + file_path (str , required): Target file path. + hash_method (HashAlgoEnum, optional): Hash method. Defaults to `HashAlgoEnum.md5`. + chunk_size (int , optional): Chunk size. Defaults to 4096. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and file doesn't exist. + + Returns: + str: File checksum. + """ + + _file_checksum: str = None + if await aiofiles.os.path.isfile(file_path): + _file_hash = hashlib.new(hash_method.value) + async with aiofiles.open(file_path, "rb") as _file: + while True: + _file_chunk = await _file.read(chunk_size) + if not _file_chunk: + break + _file_hash.update(_file_chunk) + + _file_checksum = _file_hash.hexdigest() + else: + _message = f"'{file_path}' file doesn't exist!" + if warn_mode == WarnEnum.ALWAYS: + logger.warning(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, _message) + + return _file_checksum + + +## Sync: +@validate_call +def create_dir( + create_dir: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Create directory if `create_dir` doesn't exist. + + Args: + create_dir (str, required): Create directory path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and directory already exists. + OSError: If failed to create directory. + """ + + if not os.path.isdir(create_dir): + try: + _message = f"Creating '{create_dir}' directory..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + os.makedirs(create_dir) + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{create_dir}' directory already exists!") + else: + logger.error(f"Failed to create '{create_dir}' directory!") + raise + + _message = f"Successfully created '{create_dir}' directory." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.EEXIST, f"'{create_dir}' directory already exists!") + + return + + +@validate_call +def remove_dir( + remove_dir: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Remove directory if `remove_dir` exists. + + Args: + remove_dir (str, required): Remove directory path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and directory doesn't exist. + OSError: If failed to remove directory. + """ + + if os.path.isdir(remove_dir): + try: + _message = f"Removing '{remove_dir}' directory..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + shutil.rmtree(remove_dir) + except OSError as err: + if (err.errno == errno.ENOENT) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{remove_dir}' directory doesn't exist!") + else: + logger.error(f"Failed to remove '{remove_dir}' directory!") + raise + + _message = f"Successfully removed '{remove_dir}' directory." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, f"'{create_dir}' directory doesn't exist!") + + return + + +@validate_call +def remove_dirs( + remove_dirs: List[constr(strip_whitespace=True, min_length=1, max_length=_path_max_length)], # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Remove directories if `remove_dirs` exist. + + Args: + remove_dirs (List[str], required): Remove directory paths as list. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + """ + + for _remove_dir in remove_dirs: + remove_dir(remove_dir=_remove_dir, warn_mode=warn_mode) + + return + + +@validate_call +def remove_file( + file_path: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Remove file if `file_path` exists. + + Args: + file_path (str, required): Remove file path. + warn_mode (str, optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and file doesn't exist. + OSError: If failed to remove file. + """ + + if os.path.isfile(file_path): + try: + _message = f"Removing '{file_path}' file..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + os.remove(file_path) + except OSError as err: + if (err.errno == errno.ENOENT) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{file_path}' file doesn't exist!") + else: + logger.error(f"Failed to remove '{file_path}' file!") + raise + + _message = f"Successfully removed '{file_path}' file." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, f"'{file_path}' file doesn't exist!") + + return + + +@validate_call +def remove_files( + file_paths: List[constr(strip_whitespace=True, min_length=1, max_length=_path_max_length)], # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Remove files if `file_paths` exist. + + Args: + file_paths (List[str], required): Remove file paths as list. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + """ + + for _file_path in file_paths: + remove_file(file_path=_file_path, warn_mode=warn_mode) + + return + + +@validate_call +def get_file_checksum( + file_path: constr(strip_whitespace=True, min_length=1, max_length=_path_max_length), # type: ignore + hash_method: HashAlgoEnum = HashAlgoEnum.md5, + chunk_size: conint(ge=10) = 4096, # type: ignore + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> str: + """Get file checksum. + + Args: + file_path (str , required): Target file path. + hash_method (HashAlgoEnum, optional): Hash method. Defaults to `HashAlgoEnum.md5`. + chunk_size (int , optional): Chunk size. Defaults to 4096. + warn_mode (str , optional): Warning message mode, for example: 'ERROR', 'ALWAYS', 'DEBUG', 'IGNORE'. Defaults to 'DEBUG'. + + Raises: + OSError: When warning mode is set to ERROR and file doesn't exist. + + Returns: + str: File checksum. + """ + + _file_checksum: str = None + if os.path.isfile(file_path): + _file_hash = hashlib.new(hash_method.value) + with open(file_path, "rb") as _file: + while True: + _file_chunk = _file.read(chunk_size) + if not _file_chunk: + break + _file_hash.update(_file_chunk) + + _file_checksum = _file_hash.hexdigest() + else: + _message = f"'{file_path}' file doesn't exist!" + if warn_mode == WarnEnum.ALWAYS: + logger.warning(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + elif warn_mode == WarnEnum.ERROR: + raise OSError(errno.ENOENT, _message) + + return _file_checksum + + +__all__ = [ + "async_create_dir", + "async_remove_dir", + "async_remove_dirs", + "async_remove_file", + "async_remove_files", + "async_get_file_checksum", + "create_dir", + "remove_dir", + "remove_dirs", + "remove_file", + "remove_files", + "get_file_checksum", +] diff --git a/src/bv_challenge/challenge/api/core/utils/_sanitizer.py b/src/bv_challenge/challenge/api/core/utils/_sanitizer.py new file mode 100644 index 0000000..80592a9 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_sanitizer.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +import re +import html +from urllib.parse import quote + +from pydantic import validate_call, constr, AnyHttpUrl + +from api.core.constants import ( + SPECIAL_CHARS_BASE_REGEX, + SPECIAL_CHARS_LOW_REGEX, + SPECIAL_CHARS_MEDIUM_REGEX, + SPECIAL_CHARS_HIGH_REGEX, + SPECIAL_CHARS_STRICT_REGEX, +) + + +@validate_call +def escape_html(val: constr(strip_whitespace=True)) -> str: # type: ignore + """Escape HTML characters. + + Args: + val (str, required): String to escape. + + Returns: + str: Escaped string. + """ + + _escaped = html.escape(val) + return _escaped + + +@validate_call +def espace_url(val: AnyHttpUrl) -> str: + """Escape URL characters. + + Args: + val (AnyHttpUrl, required): String to escape. + + Returns: + str: Escaped string. + """ + + _escaped = quote(val) + return _escaped + + +@validate_call +def clean_special_chars(val: str, mode: str = "LOW") -> str: + """Sanitize special characters. + + Args: + val (str, required): String to sanitize. + mode (str, optional): Sanitization mode. Defaults to "LOW". + + Raises: + ValueError: If `mode` is unsupported. + + Returns: + str: Sanitized string. + """ + + _pattern = r"" + mode = mode.upper() + if (mode == "BASE") or (mode == "HTML"): + _pattern = SPECIAL_CHARS_BASE_REGEX + elif mode == "LOW": + _pattern = SPECIAL_CHARS_LOW_REGEX + elif mode == "MEDIUM": + _pattern = SPECIAL_CHARS_MEDIUM_REGEX + elif (mode == "HIGH") or (mode == "SCRIPT") or (mode == "SQL"): + _pattern = SPECIAL_CHARS_HIGH_REGEX + elif mode == "STRICT": + _pattern = SPECIAL_CHARS_STRICT_REGEX + else: + raise ValueError(f"Unsupported mode: {mode}") + + _sanitized = re.sub(pattern=_pattern, repl="", string=val) + return _sanitized + + +__all__ = [ + "escape_html", + "espace_url", + "clean_special_chars", +] diff --git a/src/bv_challenge/challenge/api/core/utils/_secure.py b/src/bv_challenge/challenge/api/core/utils/_secure.py new file mode 100644 index 0000000..3680440 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_secure.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +import uuid +import string +import secrets +import hashlib + +from pydantic import validate_call, conint, constr + +from api.core.constants import HashAlgoEnum +from ._dt import now_ts + + +@validate_call +def gen_unique_id(prefix: constr(strip_whitespace=True, max_length=32) = "") -> str: # type: ignore + """Generate unique id. + + Args: + prefix (str, optional): Prefix of id. Defaults to ''. + + Returns: + str: Unique id. + """ + + _id = str(f"{prefix}{now_ts()}_{uuid.uuid4().hex}").lower() + return _id + + +@validate_call +def gen_random_string(length: conint(ge=1) = 16, is_alphanum: bool = True) -> str: # type: ignore + """Generate secure random string. + + Args: + length (int , optional): Length of random string. Defaults to 16. + is_alphanum (bool, optional): If True, generate only alphanumeric string. Defaults to True. + + Returns: + str: Generated random string. + """ + + _base_chars = string.ascii_letters + string.digits + if not is_alphanum: + _base_chars += string.punctuation + + _random_str = "".join(secrets.choice(_base_chars) for _i in range(length)) + return _random_str + + +@validate_call +def hash_str(val: str, algorithm: HashAlgoEnum = HashAlgoEnum.sha256) -> str: + """Hash a string using a specified hash algorithm. + + Args: + val (str , required): The string to hash. + algorithm (HashAlgoEnum, required): The hash algorithm to use. Defaults to `HashAlgoEnum.sha256`. + + Returns: + str: The hexadecimal representation of the digest. + """ + + if not isinstance(val, bytes): + val = val.encode("utf-8") + + _hash = hashlib.new(algorithm.value) + _hash.update(val) + + _hash_val = _hash.hexdigest() + return _hash_val + + +__all__ = [ + "gen_unique_id", + "gen_random_string", + "hash_str", +] diff --git a/src/bv_challenge/challenge/api/core/utils/_validator.py b/src/bv_challenge/challenge/api/core/utils/_validator.py new file mode 100644 index 0000000..2090990 --- /dev/null +++ b/src/bv_challenge/challenge/api/core/utils/_validator.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +import re +from typing import List, Union, Pattern + +from pydantic import validate_call + +from api.core.constants import ( + REQUEST_ID_REGEX, + SPECIAL_CHARS_BASE_REGEX, + SPECIAL_CHARS_LOW_REGEX, + SPECIAL_CHARS_MEDIUM_REGEX, + SPECIAL_CHARS_HIGH_REGEX, + SPECIAL_CHARS_STRICT_REGEX, +) + + +@validate_call +def is_truthy(val: Union[str, bool, int, float, None]) -> bool: + """Check if the value is truthy. + + Args: + val (Union[str, bool, int, float, None], required): Value to check. + + Raises: + ValueError: If `val` argument type is string and value is invalid. + + Returns: + bool: True if the value is truthy, False otherwise. + """ + + if isinstance(val, str): + val = val.strip().lower() + + if val in ["0", "false", "f", "no", "n", "off"]: + return False + elif val in ["1", "true", "t", "yes", "y", "on"]: + return True + else: + raise ValueError(f"`val` argument value is invalid: '{val}'!") + + return bool(val) + + +@validate_call +def is_falsy(val: Union[str, bool, int, float, None]) -> bool: + """Check if the value is falsy. + + Args: + val (Union[str, bool, int, float, None], required): Value to check. + + Returns: + bool: True if the value is falsy, False otherwise. + """ + + return not is_truthy(val) + + +@validate_call +def is_request_id(val: str) -> bool: + """Check if the string is valid request ID. + + Args: + val (str, required): String to check. + + Returns: + bool: True if the string is valid request ID, False otherwise. + """ + + _is_valid = bool(re.match(pattern=REQUEST_ID_REGEX, string=val)) + return _is_valid + + +@validate_call +def is_blacklisted(val: str, blacklist: List[str]) -> bool: + """Check if the string is blacklisted. + + Args: + val (str , required): String to check. + blacklist (List[str], required): List of blacklisted strings. + + Returns: + bool: True if the string is blacklisted, False otherwise. + """ + + for _blacklisted in blacklist: + if _blacklisted in val: + return True + + return False + + +@validate_call +def is_valid(val: str, pattern: Union[Pattern, str]) -> bool: + """Check if the string is valid with given pattern. + + Args: + val (str , required): String to check. + pattern (Union[Pattern, str], required): Pattern regex to check. + + Returns: + bool: True if the string is valid with given pattern, False otherwise. + """ + + _is_valid = bool(re.match(pattern=pattern, string=val)) + return _is_valid + + +@validate_call +def has_special_chars(val: str, mode: str = "LOW") -> bool: + """Check if the string has special characters. + + Args: + val (str, required): String to check. + mode (str, optional): Check mode. Defaults to "LOW". + + Raises: + ValueError: If `mode` is unsupported. + + Returns: + bool: True if the string has special characters, False otherwise. + """ + + _has_special_chars = False + + _pattern = r"" + mode = mode.upper() + if (mode == "BASE") or (mode == "HTML"): + _pattern = SPECIAL_CHARS_BASE_REGEX + elif mode == "LOW": + _pattern = SPECIAL_CHARS_LOW_REGEX + elif mode == "MEDIUM": + _pattern = SPECIAL_CHARS_MEDIUM_REGEX + elif (mode == "HIGH") or (mode == "SCRIPT") or (mode == "SQL"): + _pattern = SPECIAL_CHARS_HIGH_REGEX + elif mode == "STRICT": + _pattern = SPECIAL_CHARS_STRICT_REGEX + else: + raise ValueError(f"Unsupported mode: {mode}") + + _has_special_chars = bool(re.search(pattern=_pattern, string=val)) + return _has_special_chars + + +__all__ = [ + "is_truthy", + "is_falsy", + "is_request_id", + "is_blacklisted", + "is_valid", + "has_special_chars", +] diff --git a/src/bv_challenge/challenge/api/databases/__init__.py b/src/bv_challenge/challenge/api/databases/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/__init__.py b/src/bv_challenge/challenge/api/endpoints/challenge/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/eval_runner.py b/src/bv_challenge/challenge/api/endpoints/challenge/eval_runner.py new file mode 100644 index 0000000..9f1e17d --- /dev/null +++ b/src/bv_challenge/challenge/api/endpoints/challenge/eval_runner.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +"""Run-scoped /_eval attribution and scoring orchestration (framework-free). + +Extracted from ``service.py`` so the attribution + dedup decision and the +``/score`` orchestration can be unit-tested without the global ``TaskManager``, +FastAPI, ``config``, or the crypto/detector wheels. ``service.eval_bot`` and +``service.score`` are thin adapters that inject the real decrypt/scoring/runner +callables. + +Design notes: + * Attribution is by trial-decryption against the run's session keys -- it is + order-independent and run-scoped, so out-of-order callbacks and stale + callbacks from a previous run are handled without any serial pointer. + * A payload that no current session key can decrypt is *unattributable* + (stale/previous-run/garbage/tampered). It is ignored; it never consumes or + advances another session. A real session that never reports is covered by + ``RunStore.finalize`` timeout padding instead. + * ``RunStore.record`` is the atomic source of truth for "scored exactly once", + so a duplicate/replayed callback -- even one that races a live callback -- + cannot double-count. +""" + +import json +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional, Protocol, Tuple + + +class _Store(Protocol): + """Minimal RunStore surface used here (duck-typed to avoid app imports).""" + + private_keys: Dict[str, str] + + def is_completed(self, session_id: str) -> bool: ... + + def record(self, session_id: str, score: float) -> bool: ... + + def finalize(self, timeout_score: float) -> float: ... + + +@dataclass(frozen=True) +class EvalOutcome: + """Result of processing a single ``/_eval`` callback.""" + + status: str # "recorded" | "duplicate" | "unattributable" + session_id: Optional[str] = None + score: Optional[float] = None + + +def identify_session( + private_keys: Dict[str, str], + data: str, + decrypt_fn: Callable[..., str], + loads: Callable[[str], Any] = json.loads, +) -> Tuple[Optional[str], Optional[dict]]: + """Attribute an encrypted payload to its session by trial-decryption. + + Returns ``(session_id, plain_data)`` for the first session key that both + decrypts the payload and yields parseable data, else ``(None, None)``. + Completed sessions keep their keys, so replays stay attributable here and are + then deduped by the caller. + """ + for _session_id, _private_key in list(private_keys.items()): + try: + _plaintext = decrypt_fn(ciphertext=data, private_key=_private_key) + _plain_data = loads(_plaintext) + except Exception: + continue + return _session_id, _plain_data + return None, None + + +def process_eval( + store: _Store, + data: str, + *, + decrypt_fn: Callable[..., str], + score_fn: Callable[[dict], float], +) -> EvalOutcome: + """Attribute, dedup, and record exactly one ``/_eval`` callback. + + ``score_fn`` must never raise (it owns its own fallback); it is only invoked + for an attributable, not-yet-completed session. + """ + _session_id, _plain_data = identify_session(store.private_keys, data, decrypt_fn) + + if _session_id is None: + return EvalOutcome(status="unattributable") + + if store.is_completed(_session_id): + return EvalOutcome(status="duplicate", session_id=_session_id) + + _score = score_fn(_plain_data) + + # record() is atomic and rejects an already-completed session, so a callback + # that raced another for the same session collapses to "duplicate" here. + if not store.record(_session_id, _score): + return EvalOutcome(status="duplicate", session_id=_session_id) + + return EvalOutcome(status="recorded", session_id=_session_id, score=_score) + + +def run_scoring( + *, + store: _Store, + start_runner: Callable[[], Any], + wait_for_completion: Callable[[], Any], + timeout_score: float, + runner_fail_score: float, + on_runner_error: Optional[Callable[[Exception], Any]] = None, +) -> float: + """Start the runner, wait for callbacks, then finalize the run score. + + If ``start_runner`` raises, returns ``runner_fail_score`` without waiting or + touching the store. Otherwise blocks via ``wait_for_completion`` (the caller + owns the wait policy) and averages over the EXPECTED session count, padding + any session that never reported with ``timeout_score``. + """ + try: + start_runner() + except Exception as err: + if on_runner_error is not None: + on_runner_error(err) + return runner_fail_score + + wait_for_completion() + return store.finalize(timeout_score=timeout_score) + + +__all__ = ["EvalOutcome", "identify_session", "process_eval", "run_scoring"] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/router.py b/src/bv_challenge/challenge/api/endpoints/challenge/router.py index 98ffc0b..a6678bb 100644 --- a/src/bv_challenge/challenge/api/endpoints/challenge/router.py +++ b/src/bv_challenge/challenge/api/endpoints/challenge/router.py @@ -1,12 +1,18 @@ -from fastapi import APIRouter, Request, HTTPException -from fastapi.responses import JSONResponse +# -*- coding: utf-8 -*- -from api.core.constants import ErrorCodeEnum -from api.core.exceptions import BaseHTTPException +from fastapi import APIRouter, Request, HTTPException, Body +from fastapi.responses import HTMLResponse, JSONResponse + +from api.core.responses import BaseResponse +from api.endpoints.challenge.schemas import ( + MinerInput, + MinerOutput, + EvalPayload, + RandomValRequest, +) +from api.endpoints.challenge import service from api.logger import logger -from .schemas import MinerInput, MinerOutput -from . import service router = APIRouter(tags=["Challenge"]) @@ -14,7 +20,7 @@ @router.get( "/task", summary="Get task", - description="This endpoint returns the task for the miner.", + description="This endpoint returns the webpage URL for the challenge.", response_class=JSONResponse, response_model=MinerInput, ) @@ -28,14 +34,14 @@ def get_task(request: Request): _miner_input = service.get_task() logger.success(f"[{_request_id}] - Successfully got the task.") - except HTTPException: - raise - except Exception: - logger.exception(f"[{_request_id}] - Failed to get task!") - raise BaseHTTPException( - error_enum=ErrorCodeEnum.INTERNAL_SERVER_ERROR, - message="Failed to get task!", + except Exception as err: + if isinstance(err, HTTPException): + raise + + logger.error( + f"[{_request_id}] - Failed to get task!", ) + raise return _miner_input @@ -45,29 +51,140 @@ def get_task(request: Request): summary="Score", description="This endpoint score miner output.", response_class=JSONResponse, - responses={422: {}}, + responses={400: {}, 422: {}}, ) -def post_score(request: Request, miner_input: MinerInput, miner_output: MinerOutput): +def post_score( + request: Request, + miner_input: MinerInput, + miner_output: MinerOutput, +): _request_id = request.state.request_id - logger.info(f"[{_request_id}] - Scoring the miner output...") + logger.info(f"[{_request_id}] - Evaluating the miner output...") - _score: float = 0.0 try: - _score = service.score(request_id=_request_id, miner_output=miner_output) - logger.success( - f"[{_request_id}] - Successfully scored the miner output: {_score}" - ) + _score = service.score(miner_output=miner_output) except HTTPException: + # Already a well-formed HTTP error (e.g. TOO_MANY_REQUESTS) -- let it + # propagate so the client gets the real status, never a 200/null. + logger.error(f"[{_request_id}] - Failed to evaluate the miner output!") raise - except Exception: - logger.exception(f"[{_request_id}] - Failed to score the miner output!") - raise BaseHTTPException( - error_enum=ErrorCodeEnum.INTERNAL_SERVER_ERROR, - message="Failed to score the miner output!", + except Exception as err: + logger.error( + f"[{_request_id}] - Unexpected error evaluating the miner output: {err}" + ) + raise HTTPException( + status_code=500, detail="Failed to evaluate the miner output." ) + logger.success(f"[{_request_id}] - Successfully scored the miner output: {_score}") return _score +@router.get( + "/_web", + summary="Serves the webpage", + description="This endpoint serves the webpage for the challenge.", + response_class=HTMLResponse, + responses={429: {}}, +) +def _get_web(request: Request): + + _request_id = request.state.request_id + logger.info(f"[{_request_id}] - Getting webpage...") + + _html_response: HTMLResponse + try: + _html_response = service.get_web(request=request) + + logger.success(f"[{_request_id}] - Successfully got the webpage.") + except Exception as err: + if isinstance(err, HTTPException): + raise + + logger.error( + f"[{_request_id}] - Failed to get the webpage!", + ) + raise + + return _html_response + + +@router.post( + "/_random_val", + summary="Random value", + responses={401: {}, 422: {}, 429: {}}, +) +def post_random_val(request: Request, payload: RandomValRequest): + _request_id = request.state.request_id + logger.info(f"[{_request_id}] - Checking random val...") + + random_val = payload.random_val.strip() + nonce_val: str + try: + nonce_val = service.get_random_val(nonce=random_val) + logger.success(f"[{_request_id}] - Successfully checked the random val.") + except Exception as err: + if isinstance(err, HTTPException): + raise + logger.error(f"[{_request_id}] - Failed to check the random val!") + raise + + _response = {"nonce_val": nonce_val} + return _response + + +@router.post( + "/_eval", + summary="Evaluate", + description="This endpoint evaluate.", + responses={422: {}, 429: {}}, +) +def _post_eval_bot( + request: Request, + payload: EvalPayload, +): + _request_id = request.state.request_id + logger.info(f"[{_request_id}] - Evaluating the bot...") + + try: + # Extract the data from the nested structure + data = payload.error.data + service.eval_bot(data=data) + + logger.success(f"[{_request_id}] - Successfully evaluated the bot.") + except Exception as err: + if isinstance(err, HTTPException): + raise + + logger.error( + f"[{_request_id}] - Failed to evaluate the bot!", + ) + raise + + _response = BaseResponse(request=request, message="Successfully evaluated the bot.") + return _response + + +@router.post( + "/compare", + summary="Compare miner outputs (disabled)", + description="Disabled: the comparison backend is not wired into this build.", + responses={501: {}}, +) +def post_compare( + request: Request, + miner_output: dict = Body(...), + reference_output: dict = Body(...), + miner_input: dict = Body(...), +): + # Disabled rather than silently returning a misleading 0.0 similarity. Re-enable + # by wiring a real comparer into service.compare_outputs and restoring the body. + _request_id = request.state.request_id + logger.warning(f"[{_request_id}] - /compare is disabled (no comparison backend).") + raise HTTPException( + status_code=501, detail="Comparison endpoint is not available." + ) + + __all__ = ["router"] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/schemas.py b/src/bv_challenge/challenge/api/endpoints/challenge/schemas.py index 6742abf..1c2c309 100644 --- a/src/bv_challenge/challenge/api/endpoints/challenge/schemas.py +++ b/src/bv_challenge/challenge/api/endpoints/challenge/schemas.py @@ -1,57 +1,198 @@ -from pydantic import BaseModel, Field, field_validator +# -*- coding: utf-8 -*- -from potato_util.generator import gen_random_string +import os +import pathlib +from typing import Optional, Union, List, Dict, Any +from pydantic import BaseModel, Field, constr, field_validator -class MinerInput(BaseModel): - random_val: str | None = Field( - default_factory=gen_random_string, - title="Random Value", +from api.core.constants import ( + ALPHANUM_REGEX, + ALPHANUM_HOST_REGEX, + ALPHANUM_EXTEND_REGEX, + REQUIREMENTS_REGEX, + ALPHANUM_CUSTOM_REGEX, +) +from api.config import config +from api.core import utils + + +_src_dir = pathlib.Path(__file__).parent.parent.parent.parent.resolve() +_bot_dir = _src_dir / "bot" +_bot_core_dir = _bot_dir / "src" / "core" + +_bot_py_path = str(_bot_core_dir / "bot.py") +_bot_py_content = "def run_bot(driver):\n print('Hello, World!')" +if os.path.exists(_bot_py_path): + with open(_bot_py_path, "r") as _bot_py_file: + _bot_py_content = _bot_py_file.read() + +_dockerfile_path = str(_bot_dir / "Dockerfile") +# NOTE: The vm-runner strips any ENTRYPOINT/CMD and forces its own: +# cd /app && source venv/bin/activate && exec python -u main.py +# So the submitted Dockerfile MUST create a venv at /app/venv. +_dockerfile_content = """FROM redteamsubnet61/bv-bot-base:latest +COPY requirements.txt . +RUN sudo chown -R seluser:seluser /app && \\ + python3 -m venv venv && \\ + . venv/bin/activate && \\ + python3 -m pip install --no-cache-dir -r requirements.txt +COPY ./src .""" +if os.path.exists(_dockerfile_path): + with open(_dockerfile_path, "r") as _dockerfile_file: + _dockerfile_content = _dockerfile_file.read() + + + +class KeyPairPM(BaseModel): + private_key: str = Field( + ..., + min_length=32, + title="Private Key", + description="Private key as a string.", + ) + public_key: Union[str, None] = Field( + ..., + min_length=32, + title="Public Key", + description="Public key as a string.", + ) + nonce: Union[ + constr( + strip_whitespace=True, + min_length=4, + max_length=64, + pattern=ALPHANUM_REGEX, + ), # type: ignore + None, + ] = Field( + ..., + title="Nonce", description="Random value to prevent caching.", examples=["a1b2c3d4e5f6g7h8"], ) -class CommitFilePM(BaseModel): - file_name: str = Field( +class MinerFilePM(BaseModel): + fname: constr(strip_whitespace=True) = Field( # type: ignore ..., min_length=4, max_length=64, + pattern=ALPHANUM_HOST_REGEX, title="File Name", description="Name of the file.", - examples=["solution.js"], + examples=["config.py"], ) - content: str = Field( + content: constr(strip_whitespace=True) = Field( # type: ignore ..., min_length=2, title="File Content", description="Content of the file as a string.", - examples=["console.log('Challenge accepted!');"], + examples=["threshold = 0.5"], + ) + + @field_validator("fname") + @classmethod + def _check_fname(cls, val: str) -> str: + + if not isinstance(val, str): + raise TypeError("File name must be a string!") + + if val.startswith("."): + raise ValueError("File name cannot start with a dot(.)!") + + _allowed_exts = config.challenge.allowed_file_exts + if not val.endswith(tuple(_allowed_exts)): + raise ValueError( + f"File extension is not supported, only '{_allowed_exts}' extensions are allowed!" + ) + + return val + + +class MinerInput(BaseModel): + random_val: Optional[ + constr( + strip_whitespace=True, min_length=4, max_length=64, pattern=ALPHANUM_REGEX + ) # type: ignore + ] = Field( + default_factory=utils.gen_random_string, + title="Random Value", + description="Random value to prevent caching.", + examples=["a1b2c3d4e5f6g7h8"], ) class MinerOutput(BaseModel): - commit_files: list[CommitFilePM] = Field( + bot_py: str = Field( + ..., + title="bot.py", + min_length=2, + description="The main bot.py source code for the challenge.", + examples=[_bot_py_content], + ) + dockerfile: str = Field( ..., - title="Commit Files", - description="List of Commit files for the challenge.", + title="Dockerfile", + min_length=2, + description="Dockerfile to build the bot container", + examples=[_dockerfile_content], + ) + score_job_id: str = Field( + default="", + max_length=128, + description=( + "Optional caller-supplied job ID. Forwarded to vm-runner so the bot " + "container is named bot_container_ and labeled score_job_id=, " + "enabling external log streaming." + ), ) - @field_validator("commit_files", mode="after") + @field_validator("bot_py", mode="after") @classmethod - def _check_commit_files(cls, val: list[CommitFilePM]) -> list[CommitFilePM]: - for _miner_file_pm in val: - _content_lines = _miner_file_pm.content.splitlines() - if len(_content_lines) > 500: - raise ValueError( - f"`{_miner_file_pm.file_name}` file contains too many lines, should be <= 500 lines!" - ) + def _check_bot_py_lines(cls, val: str) -> str: + _lines = val.split("\n") + if len(_lines) > 2000: + raise ValueError("bot_py content is too long, max 2000 lines are allowed!") + return val + @field_validator("dockerfile", mode="after") + @classmethod + def _check_dockerfile_lines(cls, val: str) -> str: + _lines = val.split("\n") + if len(_lines) > 500: + raise ValueError("Dockerfile content is too long, max 500 lines are allowed!") return val +class ErrorData(BaseModel): + data: str = Field( + ..., + min_length=2, + pattern=ALPHANUM_CUSTOM_REGEX, + description="Bot data to evaluate.", + examples=["data"], + ) + + +class EvalPayload(BaseModel): + error: ErrorData + + +class RandomValRequest(BaseModel): + random_val: str = Field( + ..., + min_length=4, + max_length=64, + pattern=ALPHANUM_REGEX, + title="Random value", + description="Random value.", + examples=["a1b2c3d4e5f6g7h8"] + ) __all__ = [ + "KeyPairPM", "MinerInput", - "CommitFilePM", "MinerOutput", + "EvalPayload", + "RandomValRequest" ] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/scoring.py b/src/bv_challenge/challenge/api/endpoints/challenge/scoring.py new file mode 100644 index 0000000..3268cc6 --- /dev/null +++ b/src/bv_challenge/challenge/api/endpoints/challenge/scoring.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- + +"""Public scoring boundary (Layer 2). + +The detection logic itself lives in the compiled, private ``rt_bv_score`` +wheel (shipped as a binary ``.so`` like ``vault_unlock``) so the algorithm is +not readable in this public repo. This module only provides a safe wrapper: +it calls the detector, validates its result, and falls back to a neutral +score on any failure. +""" + +import logging +import math +from collections.abc import Callable +from typing import Any + +logger = logging.getLogger(__name__) + +METRICS_PROCESSOR_ERROR_SCORE = 0.5 + +try: + from rt_bv_score import MetricsProcessor as _default_metrics_processor + from rt_bv_score import gate as _default_gate +except ImportError: # private detector wheel not installed (local dev / CI) + _default_metrics_processor = None + _default_gate = None + logger.warning( + "rt_bv_score is not installed; scoring falls back to the error score " + "and the Layer 1 gate is skipped unless injected." + ) + + +# --- Layer 1 (public): structural shape validation -------------------------- +# This is deliberately *basic and non-secret*: it only checks that the decrypted +# payload has the expected containers and that the (additive) advanced-signal +# fields, when present, are the right type. It contains NO thresholds, weights, +# or anti-bot heuristics — all of that lives in the private detector. Keeping a +# cheap public shape gate here lets us reject obviously malformed / tampered +# payloads before they ever reach the private wheel. + +# Legacy raw series that every well-formed payload must carry as lists. +_REQUIRED_LIST_FIELDS = ( + "movements", + "clicks", + "mouseDowns", + "mouseUps", + "keydowns", + "keyups", + "scroll", +) +# Advanced raw signals: validated only for *shape* when present (forward/backward +# compatible — older payloads without them still pass this public gate). +_OPTIONAL_LIST_FIELDS = ("eventSequence", "targets") +_OPTIONAL_DICT_FIELDS = ("pageTimings", "trustedEventStats", "taskProgress") + + +def validate_shape(data: Any) -> tuple[bool, str | None]: + """Public Layer 1 shape check. Returns (ok, reason). + + Structural only: confirms the payload is a dict, the legacy raw series are + lists, and the advanced-signal fields (browserInfo / pageTimings / + eventSequence / targets / trustedEventStats / taskProgress) are correctly + typed when present. No behavioral judgement is made here. + """ + if not isinstance(data, dict): + return False, "payload is not an object" + + for _field in _REQUIRED_LIST_FIELDS: + if _field not in data: + return False, f"missing required field: {_field}" + if not isinstance(data[_field], list): + return False, f"field is not a list: {_field}" + + for _field in _OPTIONAL_LIST_FIELDS: + if _field in data and not isinstance(data[_field], list): + return False, f"field is not a list: {_field}" + + for _field in _OPTIONAL_DICT_FIELDS: + if _field in data and not isinstance(data[_field], dict): + return False, f"field is not an object: {_field}" + + # browserInfo is an object when present, but may legitimately be null when the + # environment snapshot was unavailable in the browser. + if "browserInfo" in data and data["browserInfo"] is not None: + if not isinstance(data["browserInfo"], dict): + return False, "field is not an object: browserInfo" + + return True, None + + +def passes_gate( + data: dict, + gate: Callable[[dict], dict] | None = None, +) -> tuple[bool, str | None]: + """Run the Layer 1 gate. Returns (passed, reason). + + Fails *closed* (rejects) on a malformed gate result or a gate error, since + the gate guards against adversarial payloads. When no gate is available + (dev/CI without the wheel) it passes through so the scorer's own fallback + governs the outcome. + """ + gate_fn = gate or _default_gate + if gate_fn is None: + return True, "gate unavailable (dev passthrough)" + + try: + result = gate_fn(data) + except Exception as err: + logger.exception("Layer 1 gate failed: %s", err) + return False, "gate error" + + if not isinstance(result, dict) or "passed" not in result: + logger.warning("Gate returned malformed result: %r", result) + return False, "gate malformed result" + + return bool(result.get("passed")), result.get("reason") + + +def score_with_metrics_processor( + data: dict, + metrics_processor: Callable[[dict], dict] | None = None, + error_score: float = METRICS_PROCESSOR_ERROR_SCORE, +) -> float: + """Run the detector on a decrypted payload and return a clamped 0..1 score.""" + processor = metrics_processor or _default_metrics_processor + if processor is None: + logger.error("No MetricsProcessor available; returning error score.") + return error_score + + try: + result = processor(data) + except Exception as err: + logger.exception("MetricsProcessor failed: %s", err) + return error_score + + if not isinstance(result, dict): + logger.warning("MetricsProcessor returned non-dict result: %r", result) + return error_score + + score = _coerce_score(result.get("score")) + if score is None: + logger.warning("MetricsProcessor returned invalid score: %r", result.get("score")) + return error_score + + score_message = result.get("score_message") + if score_message: + logger.info("MetricsProcessor score_message: %s", score_message) + + return min(max(score, 0.0), 1.0) + + +def _coerce_score(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + + try: + score = float(value) + except (TypeError, ValueError): + return None + + if math.isnan(score) or math.isinf(score): + return None + + return score + + +__all__ = ["validate_shape", "passes_gate", "score_with_metrics_processor"] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/service.py b/src/bv_challenge/challenge/api/endpoints/challenge/service.py index 488f3bb..69381bb 100644 --- a/src/bv_challenge/challenge/api/endpoints/challenge/service.py +++ b/src/bv_challenge/challenge/api/endpoints/challenge/service.py @@ -1,22 +1,406 @@ -import random +# -*- coding: utf-8 -*- + +import os +import time +import hashlib +import pathlib +import threading +from typing import List, Union, Dict, Tuple, Optional from pydantic import validate_call +from fastapi import Request +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates +# from rt_comparer import RTComparer + +# try: +# from modules.rt_bv_score import MetricsProcessor # type: ignore +# except ImportError: +# from rt_bv_score import MetricsProcessor # type: ignore + +from api.core.constants import ErrorCodeEnum +from api.core import utils +from api.config import config +from api.core.exceptions import BaseHTTPException +from api.helpers.crypto import asymmetric as asymmetric_helper +from api.endpoints.challenge.schemas import KeyPairPM, MinerInput, MinerOutput +from api.endpoints.challenge import utils as ch_utils +from api.endpoints.challenge import scoring +from api.endpoints.challenge import eval_runner +from api.endpoints.challenge.session_store import RunStore +from api.logger import logger + + +_src_dir = pathlib.Path(__file__).parent.parent.parent.parent.resolve() + + +class TaskManager: + """ + Task Manager for handling key pairs, action lists, and evaluation metrics + during challenge sessions. + """ + + @validate_call + def __init__(self, uid: str = None): + self.uid = uid + # Serializes whole scoring runs so concurrent /score calls cannot + # clobber each other's run store. + self.run_lock = threading.Lock() + # Serializes the per-session key handoff (claim of the next session via + # /_random_val). Distinct from run_lock, which /score holds for the whole + # run -- the handoff happens *during* that hold, so it needs its own lock. + self.claim_lock = threading.Lock() + self.reset_tasks() + + def reset_tasks(self) -> None: + """Reset all tasks: regenerate key pairs, action lists, and run store.""" + self._actions_idx = 0 + + # Generate key pairs (one per session) + self.key_pairs = ch_utils.gen_key_pairs( + n_challenge=config.challenge.n_run_per_ch, + key_size=config.api.security.asymmetric.key_size, + ) + + # Generate challenge actions + self.challenges_action_list = ch_utils.gen_cb_actions( + n_challenge=config.challenge.n_ch_per_epoch, + window_width=config.challenge.window_width, + window_height=config.challenge.window_height, + n_checkboxes=config.challenge.n_checkboxes, + min_distance=config.challenge.cb_min_distance, + max_factor=config.challenge.cb_gen_max_factor, + checkbox_size=config.challenge.cb_size, + exclude_areas=config.challenge.cb_exclude_areas, + pre_action_list=config.challenge.cb_pre_action_list, + ) + + # Build the run store. session_id = per-session nonce. Capture private + # keys now, before they are consumed/nulled during the session flow, so + # /_eval can attribute payloads by trial-decryption. + self.run_id = utils.gen_random_string(length=16) + _sessions: List[Tuple[str, str]] = [ + (kp.nonce, kp.private_key) for kp in self.key_pairs + ] + self.run_store = RunStore.create(run_id=self.run_id, sessions=_sessions) + + # Reset current task properties + self.cur_key_pair = None + self.cur_session_id: Optional[str] = None + self.cur_score = None + + def pop_task(self) -> Union[KeyPairPM, None]: + """Advance to the next session (key pair); capture its session id.""" + if not self.key_pairs: + self.cur_key_pair = None + self.cur_session_id = None + return None + + self.cur_key_pair = self.key_pairs.pop(0) + # Capture the session id now; the nonce field gets nulled later. + self.cur_session_id = self.cur_key_pair.nonce + return self.cur_key_pair -from .schemas import MinerInput, MinerOutput + def has_remaining_tasks(self) -> bool: + """Check if there are remaining tasks""" + return len(self.key_pairs) > 0 + + def get_remaining_task_count(self) -> int: + """Get the number of remaining tasks""" + return len(self.key_pairs) + + def get_nonce(self) -> str: + _nonce_key: str = self.cur_key_pair.public_key + self.cur_key_pair.public_key = None + self.cur_key_pair.nonce = None + return _nonce_key + + def get_session_info(self) -> Dict: + """Get information about current session for VM execution""" + return { + "total_sessions": config.challenge.n_run_per_ch, + "nonce": self.cur_key_pair.nonce if self.cur_key_pair else None, + } + + +# Initialize the task manager as a global variable +global tm +tm = TaskManager() def get_task() -> MinerInput: - return MinerInput() + """Get the task for the miner""" + _miner_input = MinerInput() + return _miner_input @validate_call -def score(request_id: str, miner_output: MinerOutput) -> float: +def score(miner_output: MinerOutput) -> float: + """Run one scoring run. + + Orchestration: create a run (run_id + N session ids) -> start the bot + sessions via the runner -> wait for each session to report via /_eval -> + average over the EXPECTED session count (missing sessions count as + session_timeout_score). The run lock serializes concurrent /score calls. + """ + _expected = config.challenge.n_run_per_ch + _num_tasks = config.challenge.n_ch_per_epoch * _expected + + with tm.run_lock: + # Start a fresh run if the previous one is exhausted. + if (not tm.has_remaining_tasks()) or ( + tm.get_remaining_task_count() < _num_tasks + ): + tm.reset_tasks() + + _run_store = tm.run_store + _run_id = tm.run_id + + # Claim the first session for the page-serving flow. + task = tm.pop_task() + tm.cur_score = None + if not task: + raise BaseHTTPException( + error_enum=ErrorCodeEnum.TOO_MANY_REQUESTS, + message="No initialized key pairs or action lists, or out of tasks!", + ) + + def _start_runner() -> None: + logger.info( + f"[run {_run_id}] Starting {_expected} bot session(s) via runner..." + ) + ch_utils.send_build_and_run_request( + vm_endpoint=config.challenge.vm_endpoint, + bot_py=miner_output.bot_py, + dockerfile=miner_output.dockerfile, + session_count=_expected, + timeout=config.challenge.vm_timeout, + ssl_verify=config.challenge.vm_ssl_verify, + score_job_id=miner_output.score_job_id, + ) + + def _wait_for_completion() -> None: + # Wait for sessions to report via /_eval, bounded by bot_timeout. + _i = 0 + while ( + _run_store.completed_count() < _expected + and _i < config.challenge.bot_timeout + ): + logger.info( + f"[run {_run_id}] Waiting... " + f"{_run_store.completed_count()}/{_expected} sessions recorded" + ) + time.sleep(1) + _i += 1 + + def _on_runner_error(err: Exception) -> None: + logger.error( + f"[run {_run_id}] Runner failed: {err}; returning runner_fail_score." + ) + + # Average over EXPECTED sessions; incomplete ones count as timeout. + _score = eval_runner.run_scoring( + store=_run_store, + start_runner=_start_runner, + wait_for_completion=_wait_for_completion, + timeout_score=config.challenge.session_timeout_score, + runner_fail_score=config.challenge.runner_fail_score, + on_runner_error=_on_runner_error, + ) + logger.info( + f"[run {_run_id}] Final score (avg over {_expected} sessions): {_score}" + ) + return _score + + +# Schema version of the non-behavioral browser/runtime integrity payload the +# SDK emits. Bump when the collected payload shape changes. +SCHEMA_VERSION = "bv-runtime-1" + + +def _short_digest(*parts: str) -> str: + """Stable short hex fingerprint over the given parts (non-secret).""" + _hasher = hashlib.sha256() + for _part in parts: + _hasher.update((_part or "").encode("utf-8")) + _hasher.update(b"\x00") + return _hasher.hexdigest()[:16] + + +@validate_call(config={"arbitrary_types_allowed": True}) +def get_web(request: Request) -> HTMLResponse: + """Serve the minimal browser-verification page. + + The page only loads the SDK, which collects non-behavioral browser/runtime + integrity signals and submits the encrypted payload to ``/_eval``. The + backend injects the per-session public key (encryption key material) plus + non-secret session-binding fields. + """ + # Serve the CURRENT session's public key so the SDK encrypts with a key + # whose private half lives in the run store (store.private_keys) and can + # decrypt the /_eval payload. Then advance the claim pointer so the next + # session's page load gets the next key. score() pre-claims the first one, + # so sequential /_web loads serve session keys 1..N in order. + with tm.claim_lock: + _cur = tm.cur_key_pair + if _cur and _cur.public_key: + _nonce = _cur.nonce + _public_key = _cur.public_key + tm.pop_task() # advance to the next session for the next /_web load + else: + _nonce = utils.gen_random_string() + _public_key = asymmetric_helper.gen_key_pair( + key_size=config.api.security.asymmetric.key_size, as_str=True + )[1] + logger.warning( + "/_web called with no active session key; serving a throwaway key " + "(this endpoint shouldn't be called directly outside a run)." + ) + + _public_key_id = _short_digest(_public_key) + _config_hash = _short_digest( + SCHEMA_VERSION, + str(config.api.security.asymmetric.key_size), + ) + + _templates = Jinja2Templates(directory=(_src_dir / "./templates/html")) + _html_response = _templates.TemplateResponse( + request=request, + name="index.html", + context={ + "session_id": _nonce, + "nonce": _nonce, + "public_key": _public_key, + "public_key_id": _public_key_id, + "config_hash": _config_hash, + "schema_version": SCHEMA_VERSION, + }, + ) + return _html_response + + +@validate_call +def get_random_val(nonce: str) -> str: + """Hand out the current session's public key and advance to the next session. + + This is the per-session *claim* point: validating the nonce, returning the + key, and advancing the pointer are done atomically under ``claim_lock`` so + that the serial key handoff is correct without ``/_eval`` ever touching the + pointer. Attribution of the eventual callback is independent of this pointer + (it is done by trial-decryption in ``eval_bot``). + """ + with tm.claim_lock: + if not tm.cur_key_pair: + raise BaseHTTPException( + error_enum=ErrorCodeEnum.BAD_REQUEST, + message="Not initialized key pair or out of key pair, this endpoint is shouldn't be called directly!", + ) + + if tm.cur_key_pair.nonce != nonce: + raise BaseHTTPException( + error_enum=ErrorCodeEnum.UNAUTHORIZED, + message="Invalid nonce value!", + ) + + if not tm.cur_key_pair.public_key: + raise BaseHTTPException( + error_enum=ErrorCodeEnum.TOO_MANY_REQUESTS, + message="Nonce is already retrieved!", + ) + + _nonce_key = tm.get_nonce() + # Advance the claim pointer to the next session for the next bot. The + # session just handed out stays in run_store and is attributed by its + # key at /_eval time. + tm.pop_task() + return _nonce_key + + +def _score_payload(plain_data: dict) -> float: + """Layer 1 gate then Layer 2 scorer for one decrypted payload. + + Never raises: any unexpected error falls back to the configured error score + so ``eval_bot`` can always record a result for an attributed session. + """ + try: + _shape_ok, _shape_reason = scoring.validate_shape(plain_data) + if not _shape_ok: + logger.info(f"Layer 1 shape check rejected session: {_shape_reason}") + return config.challenge.gate_fail_score + _passed, _reason = scoring.passes_gate(plain_data) + if not _passed: + logger.info(f"Layer 1 gate rejected session: {_reason}") + return config.challenge.gate_fail_score + return scoring.score_with_metrics_processor( + data=plain_data, + error_score=config.challenge.metrics_processor_error_score, + ) + except Exception as err: + logger.error(f"Unexpected scoring error; recording error_score: {err}") + return config.challenge.metrics_processor_error_score + + +@validate_call +def eval_bot(data: str) -> None: + """Evaluate exactly one browser session callback. + + Pure attribution + record -- it does NOT touch the session-claim pointer + (``cur_key_pair``/``pop_task``); that lives in ``get_random_val``. + + Invariants: + * Records each session's score EXACTLY ONCE; a duplicate/replayed callback + is ignored and never double-counts (``RunStore.record`` is the atomic + source of truth). + * An unattributable payload (no current session key decrypts it -- + stale/previous-run/garbage/tampered) is logged and ignored; it never + consumes or advances another session. A real session that never reports + is handled by ``RunStore.finalize`` timeout padding instead. + * Never raises; returns nothing (the endpoint returns a generic message + only -- the real score is never exposed). + """ + _store = tm.run_store + if _store is None: + logger.warning("eval_bot called with no active run; ignoring.") + return + + _outcome = eval_runner.process_eval( + _store, + data, + decrypt_fn=ch_utils.decrypt, + score_fn=_score_payload, + ) + + if _outcome.status == "recorded": + logger.info(f"Recorded session {_outcome.session_id} score: {_outcome.score}") + elif _outcome.status == "duplicate": + logger.warning( + f"Duplicate /_eval for session {_outcome.session_id}; not re-recording." + ) + else: # "unattributable" + logger.warning( + "Unattributable /_eval payload (stale/garbage/tampered); ignoring." + ) + + return + + +def compare_outputs(miner_input, miner_output, reference_output) -> dict: + """Disabled: the RTComparer backend is not wired into this build. - _score_result = random.random() # nosec B311 - return _score_result + The previous implementation never actually invoked a comparer (it built a + string literal and then called ``.get`` on it), so every call silently + returned ``similarity_score: 0.0``. Until a real comparer is integrated this + raises so the caller surfaces an honest "not implemented" instead of a + misleading zero score. + """ + raise NotImplementedError("compare_outputs is not available in this build") __all__ = [ "get_task", + "get_web", + "get_random_val", "score", + "eval_bot", + "compare_outputs", ] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/session_store.py b/src/bv_challenge/challenge/api/endpoints/challenge/session_store.py new file mode 100644 index 0000000..8add59d --- /dev/null +++ b/src/bv_challenge/challenge/api/endpoints/challenge/session_store.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +"""Per-run session tracking for the scoring orchestration. + +Replaces the bare global ``scores`` list with explicit run/session records so a +scoring run can be reasoned about as a unit and we can correctly handle: + + * duplicate ``/_eval`` (record a session score exactly once) + * missing sessions / session timeout (pad to the expected count) + * aggregation over the EXPECTED session count, not just completed ones + +A session is identified by its ``session_id`` (the per-session nonce). The run +keeps each session's private key so ``/_eval`` can attribute an encrypted +payload to its session by trial-decryption, independent of arrival order. +""" + +import threading +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + + +@dataclass +class SessionRecord: + session_id: str + score: Optional[float] = None + completed: bool = False + timed_out: bool = False + + +@dataclass +class RunStore: + run_id: str + expected_sessions: int + sessions: Dict[str, SessionRecord] = field(default_factory=dict) + # session_id -> private key PEM (kept for trial-decryption + dedup) + private_keys: Dict[str, str] = field(default_factory=dict) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + @classmethod + def create(cls, run_id: str, sessions: List[Tuple[str, str]]) -> "RunStore": + """Build a run from (session_id, private_key) pairs.""" + store = cls(run_id=run_id, expected_sessions=len(sessions)) + for session_id, private_key in sessions: + store.sessions[session_id] = SessionRecord(session_id=session_id) + store.private_keys[session_id] = private_key + return store + + def is_completed(self, session_id: str) -> bool: + rec = self.sessions.get(session_id) + return bool(rec and rec.completed) + + def get_score(self, session_id: str) -> Optional[float]: + rec = self.sessions.get(session_id) + return rec.score if rec else None + + def record(self, session_id: str, score: float) -> bool: + """Record a session score exactly once. + + Returns True if recorded, False if the session is unknown or already + completed (duplicate). + """ + with self._lock: + rec = self.sessions.get(session_id) + if rec is None or rec.completed: + return False + rec.score = score + rec.completed = True + return True + + def completed_count(self) -> int: + return sum(1 for rec in self.sessions.values() if rec.completed) + + def finalize(self, timeout_score: float) -> float: + """Average over EXPECTED sessions; any incomplete session counts as + ``timeout_score`` and is marked timed out.""" + if self.expected_sessions <= 0: + return 0.0 + + with self._lock: + total = 0.0 + for rec in self.sessions.values(): + if rec.completed and rec.score is not None: + total += rec.score + else: + rec.timed_out = True + total += timeout_score + return total / self.expected_sessions + + +__all__ = ["SessionRecord", "RunStore"] diff --git a/src/bv_challenge/challenge/api/endpoints/challenge/utils.py b/src/bv_challenge/challenge/api/endpoints/challenge/utils.py new file mode 100644 index 0000000..135faf5 --- /dev/null +++ b/src/bv_challenge/challenge/api/endpoints/challenge/utils.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- + +import os +import re +import time +import shutil +import random +import requests +import subprocess +from datetime import datetime, timezone +from typing import List, Dict, Union, Tuple, Optional + +import vault_unlock +from api.config import config +import docker +from docker.models.networks import Network +from docker import DockerClient +from pydantic import validate_call + +from api.core.constants import ErrorCodeEnum, ENV_PREFIX +from api.core import utils +from api.core.exceptions import BaseHTTPException +from api.helpers.crypto import asymmetric as asymmetric_helper +from api.endpoints.challenge.schemas import KeyPairPM, MinerOutput +from api.logger import logger + + +@validate_call +def gen_key_pairs(n_challenge: int, key_size: int) -> List[KeyPairPM]: + + _key_pairs: List[KeyPairPM] = [] + for _ in range(n_challenge): + _key_pair: Tuple[str, str] = asymmetric_helper.gen_key_pair( + key_size=key_size, as_str=True + ) + _private_key, _public_key = _key_pair + _nonce = utils.gen_random_string(length=32) + _key_pair_pm = KeyPairPM( + private_key=_private_key, public_key=_public_key, nonce=_nonce + ) + _key_pairs.append(_key_pair_pm) + + return _key_pairs + +@validate_call +def gen_cb_actions( + n_challenge: int = 10, + window_width: int = 1420, + window_height: int = 740, + n_checkboxes: int = 5, + min_distance: int = 300, + max_factor: int = 10, + checkbox_size: int = 20, # Assuming checkbox size ~20px + exclude_areas: Union[List[Dict[str, int]], None] = None, + pre_action_list: Union[ + List[Dict[str, Union[int, str, Dict[str, Dict[str, int]]]]], None + ] = None, +) -> List[Dict[str, Union[int, str, Dict[str, Dict[str, int]]]]]: + + _max_attempts = n_checkboxes * max_factor # Avoid infinite loops + + _challenge_list = [] + for _ in range(n_challenge): + _n_attempts = 0 + _i = 0 + _action_list = [] + + if pre_action_list: + _action_list = pre_action_list + while len(_action_list) < n_checkboxes: + _x = random.randint(checkbox_size, window_width - checkbox_size) + _y = random.randint(checkbox_size, window_height - checkbox_size) + + _is_near = False + _i = len(_action_list) + for _action in _action_list: + if _action["type"] == "click": + ## Calculate distance between two points using Euclidean distance: + if (_x - _action["args"]["location"]["x"]) ** 2 + ( + _y - _action["args"]["location"]["y"] + ) ** 2 < min_distance**2: + _is_near = True + break + + _is_in_area = False + if exclude_areas: + for _area in exclude_areas: + if (_area["x1"] <= _x <= _area["x2"]) and ( + _area["y1"] <= _y <= _area["y2"] + ): + _is_in_area = True + break + + if (not _is_near) and (not _is_in_area): + _action = { + "id": _i, + "type": "click", + "args": {"location": {"x": _x, "y": _y}}, + } + _action_list.append(_action) + + _n_attempts += 1 + + if _max_attempts <= _n_attempts: + logger.warning("Skipped generating positions due to max attempts!") + break + _next_id = len(_action_list) + _action_list.extend( + [ + { + "id": _next_id, + "type": "input", + "selector": { + "name": "username", + "id": utils.gen_random_string(length=32), + }, + "args": {"text": utils.gen_random_string(length=32)}, + }, + { + "id": _next_id + 1, + "type": "input", + "selector": { + "name": "password", + "id": utils.gen_random_string(length=32), + }, + "args": {"text": utils.gen_random_string(length=32)}, + }, + ] + ) + _challenge_list.append(_action_list) + + return _challenge_list + +@validate_call +def decrypt(ciphertext: str, private_key: str) -> str: + + _plaintext: str = vault_unlock.decrypt_payload( + encrypted_text=ciphertext, private_key_pem=private_key + ) + return _plaintext + + +@validate_call +def send_build_and_run_request( + vm_endpoint: str, + bot_py: str, + dockerfile: str, + session_count: int, + timeout: int = 120, + ssl_verify: bool = True, + score_job_id: str = "", +) -> Dict: + """ + Send build and run request to external VM. + + Args: + vm_endpoint: VM endpoint URL + bot_py: Bot Python code + dockerfile: Dockerfile content + session_count: Number of sessions to run + timeout: Request timeout in seconds + ssl_verify: Whether to verify SSL certificates + + Returns: + Response from VM containing session results + + Raises: + requests.RequestException: If request fails + ValueError: If response is invalid + """ + logger.info(f"Sending build and run request to VM: {vm_endpoint}") + + try: + _payload = { + "bot_py": bot_py, + "dockerfile": dockerfile, + "session_count": session_count, + "score_job_id": score_job_id, + } + + _response = requests.post( + f"{vm_endpoint}/build_and_run", + json=_payload, + timeout=timeout, + verify=ssl_verify, + ) + + if _response.status_code != 200: + logger.error( + f"VM request failed with status {_response.status_code}: {_response.text}" + ) + raise ValueError( + f"VM request failed with status {_response.status_code}: {_response.text}" + ) + + _result = _response.json() + logger.success("Successfully received response from VM") + return _result + + except requests.Timeout: + logger.error(f"VM request timed out after {timeout} seconds") + raise + except requests.RequestException as err: + logger.error(f"VM request failed: {str(err)}") + raise + except Exception as err: + logger.error(f"Unexpected error during VM request: {str(err)}") + raise + + +__all__ = [ + "gen_key_pairs", + "decrypt", + "send_build_and_run_request", +] diff --git a/src/bv_challenge/challenge/api/exception.py b/src/bv_challenge/challenge/api/exception.py index 0089a51..e462e88 100644 --- a/src/bv_challenge/challenge/api/exception.py +++ b/src/bv_challenge/challenge/api/exception.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from pydantic import validate_call from fastapi import FastAPI, HTTPException from fastapi.exceptions import RequestValidationError @@ -24,7 +26,7 @@ def add_exception_handlers(app: FastAPI) -> None: app.add_exception_handler(500, server_error_handler) app.add_exception_handler(HTTPException, http_exception_handler) app.add_exception_handler(RequestValidationError, validation_error_handler) - # Add more exception handlers here... + ## Add more exception handlers here... return diff --git a/src/bv_challenge/challenge/api/helpers/__init__.py b/src/bv_challenge/challenge/api/helpers/__init__.py index e69de29..40a96af 100644 --- a/src/bv_challenge/challenge/api/helpers/__init__.py +++ b/src/bv_challenge/challenge/api/helpers/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/helpers/crypto/__init__.py b/src/bv_challenge/challenge/api/helpers/crypto/__init__.py new file mode 100644 index 0000000..40a96af --- /dev/null +++ b/src/bv_challenge/challenge/api/helpers/crypto/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/src/bv_challenge/challenge/api/helpers/crypto/asymmetric.py b/src/bv_challenge/challenge/api/helpers/crypto/asymmetric.py new file mode 100644 index 0000000..6c4b95a --- /dev/null +++ b/src/bv_challenge/challenge/api/helpers/crypto/asymmetric.py @@ -0,0 +1,626 @@ +# -*- coding: utf-8 -*- + +import os +import errno +import base64 +from typing import Tuple, Union + +import aiofiles +from cryptography.hazmat.primitives.asymmetric import rsa, padding +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric.types import ( + PrivateKeyTypes, + PublicKeyTypes, +) +from pydantic import validate_call +from beans_logging import logger + +from api.core.constants import WarnEnum +from api.core import utils + + +@validate_call +def gen_key_pair( + key_size: int, + as_str: bool = False, +) -> Tuple[Union[PrivateKeyTypes, str], Union[PublicKeyTypes, str]]: + + _private_key: PrivateKeyTypes = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + _public_key: PublicKeyTypes = _private_key.public_key() + + if as_str: + _private_key: bytes = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + + _public_key: bytes = _public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode() + + return _private_key, _public_key + + +@validate_call +async def async_create_keys( + asymmetric_keys_dir: str, + key_size: int, + private_key_fname: str, + public_key_fname: str, + force: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Async generate and create asymmetric key files. + + Args: + asymmetric_keys_dir (str , required): Asymmetric keys directory. + key_size (int , required): Asymmetric key size. + private_key_fname (str , required): Asymmetric private key filename. + public_key_fname (str , required): Asymmetric public key filename. + force (bool , optional): Force to create asymmetric keys. Defaults to False. + warn_mode (WarnEnum, optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + FileExistsError: If warning mode is ERROR and asymmetric keys already exist. + OSError : If failed to create asymmetric keys. + """ + + _private_key_path = os.path.join(asymmetric_keys_dir, private_key_fname) + _public_key_path = os.path.join(asymmetric_keys_dir, public_key_fname) + + if force: + await utils.async_remove_file(file_path=_private_key_path, warn_mode=warn_mode) + await utils.async_remove_file(file_path=_public_key_path, warn_mode=warn_mode) + + if (await aiofiles.os.path.isfile(_private_key_path)) and ( + await aiofiles.os.path.isfile(_public_key_path) + ): + logger.trace( + f"Asymmetric keys already exist: ['{_private_key_path}', '{_public_key_path}']" + ) + return + + _message = ( + f"Generating asymmetric keys: ['{_private_key_path}', '{_public_key_path}']..." + ) + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _private_key: PrivateKeyTypes + if await aiofiles.os.path.isfile(_private_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_private_key_path}' private key already exists!") + + _private_key: PrivateKeyTypes = await async_get_private_key( + private_key_path=_private_key_path + ) + else: + _private_key: PrivateKeyTypes = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + + if await aiofiles.os.path.isfile(_public_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_public_key_path}' public key already exists!") + + await utils.async_remove_file(file_path=_public_key_path, warn_mode=warn_mode) + + _public_key = _private_key.public_key() + + _private_pem = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + _public_pem = _public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + + await utils.async_create_dir(create_dir=asymmetric_keys_dir, warn_mode=warn_mode) + + if not await aiofiles.os.path.isfile(_private_key_path): + try: + async with aiofiles.open(_private_key_path, "wb") as _private_key_file: + await _private_key_file.write(_private_pem) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_private_key_path}' private key already exists!") + else: + logger.error(f"Failed to create '{_private_key_path}' private key!") + raise + + if not await aiofiles.os.path.isfile(_public_key_path): + try: + async with aiofiles.open(_public_key_path, "wb") as _public_key_file: + await _public_key_file.write(_public_pem) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_public_key_path}' public key already exists!") + else: + logger.error(f"Failed to create '{_public_key_path}' public key!") + raise + + _message = f"Successfully generated asymmetric keys: ['{_private_key_path}', '{_public_key_path}']" + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + return + + +@validate_call +async def async_get_private_key( + private_key_path: str, as_str: bool = False +) -> Union[PrivateKeyTypes, str]: + """Async read asymmetric private key from file. + + Args: + private_key_path (str , required): Asymmetric private key path. + as_str (bool, optional): Return private key as string. Defaults to False. + + Raises: + FileNotFoundError: If Asymmetric private key file not found. + + Returns: + Union[PrivateKeyTypes, str]: Asymmetric private key. + """ + + if not await aiofiles.os.path.isfile(private_key_path): + raise FileNotFoundError(f"Not found '{private_key_path}' private key!") + + logger.debug(f"Reading '{private_key_path}' private key...") + _private_key: PrivateKeyTypes + async with aiofiles.open(private_key_path, "rb") as _private_key_file: + _private_key_bytes: bytes = await _private_key_file.read() + _private_key: PrivateKeyTypes = serialization.load_pem_private_key( + data=_private_key_bytes, password=None + ) + + if as_str: + _private_key = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + + logger.debug(f"Successfully read '{private_key_path}' private key.") + + return _private_key + + +@validate_call +async def async_get_public_key( + public_key_path: str, as_str: bool = False +) -> Union[PublicKeyTypes, str]: + """Async read asymmetric public key from file. + + Args: + public_key_path (str , required): Asymmetric public key path. + as_str (bool, optional): Return public key as string. Defaults to False. + + Raises: + FileNotFoundError: If asymmetric public key file not found. + + Returns: + Union[PublicKeyTypes, str]: Asymmetric public key. + """ + + if not await aiofiles.os.path.isfile(public_key_path): + raise FileNotFoundError(f"Not found '{public_key_path}' public key!") + + logger.debug(f"Reading '{public_key_path}' public key...") + _public_key: PublicKeyTypes + async with aiofiles.open(public_key_path, "rb") as _public_key_file: + _public_key_bytes: bytes = await _public_key_file.read() + _public_key: PublicKeyTypes = serialization.load_pem_public_key( + data=_public_key_bytes + ) + + if as_str: + _public_key = _public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode() + + logger.debug(f"Successfully read '{public_key_path}' public key.") + + return _public_key + + +@validate_call +async def async_get_keys( + private_key_path: str, public_key_path: str, as_str: bool = False +) -> Tuple[Union[PrivateKeyTypes, str], Union[PublicKeyTypes, str]]: + """Async read asymmetric keys from file. + + Args: + private_key_path (str , required): Asymmetric private key path. + public_key_path (str , required): Asymmetric public key path. + as_str (bool, optional): Return keys as strings. Defaults to False. + + Returns: + Tuple[Union[PrivateKeyTypes, str], Union[PublicKeyTypes, str]]: Private and public keys. + """ + + _private_key = await async_get_private_key( + private_key_path=private_key_path, as_str=as_str + ) + _public_key = await async_get_public_key( + public_key_path=public_key_path, as_str=as_str + ) + + return _private_key, _public_key + + +@validate_call +def create_keys( + asymmetric_keys_dir: str, + key_size: int, + private_key_fname: str, + public_key_fname: str, + force: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Generate and create asymmetric key files. + + Args: + asymmetric_keys_dir (str , required): Asymmetric keys directory. + key_size (int , required): Asymmetric key size. + private_key_fname (str , required): Asymmetric private key filename. + public_key_fname (str , required): Asymmetric public key filename. + force (bool , optional): Force to create asymmetric keys. Defaults to False. + warn_mode (WarnEnum, optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + FileExistsError: If warning mode is ERROR and asymmetric keys already exist. + OSError : If failed to create asymmetric keys. + """ + + _private_key_path = os.path.join(asymmetric_keys_dir, private_key_fname) + _public_key_path = os.path.join(asymmetric_keys_dir, public_key_fname) + + if force: + utils.remove_file(file_path=_private_key_path, warn_mode=warn_mode) + utils.remove_file(file_path=_public_key_path, warn_mode=warn_mode) + + if os.path.isfile(_private_key_path) and os.path.isfile(_public_key_path): + logger.trace( + f"Asymmetric keys already exist: ['{_private_key_path}', '{_public_key_path}']" + ) + return + + _message = ( + f"Generating asymmetric keys: ['{_private_key_path}', '{_public_key_path}']..." + ) + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _private_key: PrivateKeyTypes + if os.path.isfile(_private_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_private_key_path}' private key already exists!") + + _private_key: PrivateKeyTypes = get_private_key( + private_key_path=_private_key_path + ) + else: + _private_key = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + + if os.path.isfile(_public_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_public_key_path}' public key already exists!") + + utils.remove_file(file_path=_public_key_path, warn_mode=warn_mode) + + _public_key = _private_key.public_key() + + _private_pem = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + _public_pem = _public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + + utils.create_dir(create_dir=asymmetric_keys_dir, warn_mode=warn_mode) + + if not os.path.isfile(_private_key_path): + try: + with open(_private_key_path, "wb") as _private_key_file: + _private_key_file.write(_private_pem) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_private_key_path}' private key already exists!") + else: + logger.error(f"Failed to create '{_private_key_path}' private key!") + raise + + if not os.path.isfile(_public_key_path): + try: + with open(_public_key_path, "wb") as _public_key_file: + _public_key_file.write(_public_pem) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_public_key_path}' public key already exists!") + else: + logger.error(f"Failed to create '{_public_key_path}' public key!") + raise + + _message = f"Successfully generated asymmetric keys: ['{_private_key_path}', '{_public_key_path}']" + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + return + + +@validate_call +def get_private_key( + private_key_path: str, as_str: bool = False +) -> Union[PrivateKeyTypes, str]: + """Read asymmetric private key from file. + + Args: + private_key_path (str , required): Asymmetric private key path. + as_str (bool, optional): Return private key as string. Defaults to False. + + Raises: + FileNotFoundError: If asymmetric private key file not found. + + Returns: + Union[PrivateKeyTypes, str]: Asymmetric private key as PrivateKeyTypes or str. + """ + + if not os.path.isfile(private_key_path): + raise FileNotFoundError(f"Not found '{private_key_path}' private key!") + + logger.debug(f"Reading '{private_key_path}' private key...") + _private_key: PrivateKeyTypes + with open(private_key_path, "rb") as _private_key_file: + _private_key_bytes: bytes = _private_key_file.read() + _private_key: PrivateKeyTypes = serialization.load_pem_private_key( + data=_private_key_bytes, password=None + ) + + if as_str: + _private_key = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode() + + logger.debug(f"Successfully read '{private_key_path}' private key.") + + return _private_key + + +@validate_call +def get_public_key( + public_key_path: str, as_str: bool = False +) -> Union[PublicKeyTypes, str]: + """Read asymmetric public key from file. + + Args: + public_key_path (str , required): Asymmetric public key path. + as_str (bool, optional): Return public key as string. Defaults to False. + + Raises: + FileNotFoundError: If asymmetric public key file not found. + + Returns: + Union[PublicKeyTypes, str]: Asymmetric public key as PublicKeyTypes or str. + """ + + if not os.path.isfile(public_key_path): + raise FileNotFoundError(f"Not found '{public_key_path}' public key!") + + logger.debug(f"Reading '{public_key_path}' public key...") + _public_key: PublicKeyTypes + with open(public_key_path, "rb") as _public_key_file: + _public_key_bytes: bytes = _public_key_file.read() + _public_key: PublicKeyTypes = serialization.load_pem_public_key( + data=_public_key_bytes + ) + + if as_str: + _public_key = _public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode() + + logger.debug(f"Successfully read '{public_key_path}' public key.") + + return _public_key + + +@validate_call +def get_keys( + private_key_path: str, public_key_path: str, as_str: bool = False +) -> Tuple[Union[PrivateKeyTypes, str], Union[PublicKeyTypes, str]]: + """Read asymmetric keys from file. + + Args: + private_key_path (str , required): Asymmetric private key path. + public_key_path (str , required): Asymmetric public key path. + as_str (bool, optional): Return keys as strings. Defaults to False. + + Returns: + Tuple[Union[PrivateKeyTypes, str], Union[PublicKeyTypes, str]]: Private and public keys. + """ + + _private_key = get_private_key(private_key_path=private_key_path, as_str=as_str) + _public_key = get_public_key(public_key_path=public_key_path, as_str=as_str) + + return _private_key, _public_key + + +@validate_call(config={"arbitrary_types_allowed": True}) +def encrypt_with_public_key( + plaintext: Union[str, bytes], + public_key: PublicKeyTypes, + base64_encode: bool = False, + as_str: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> Union[str, bytes]: + """Encrypt plaintext with public key. + + Args: + plaintext (Union[str, bytes], required): Plaintext to encrypt. + public_key (PublicKeyTypes , required): Public key. + base64_encode (bool , optional): Encode ciphertext with base64. Defaults to False. + as_str (bool , optional): Return ciphertext as string or bytes. Defaults to False. + warn_mode (WarnEnum , optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + Exception: If failed to encrypt plaintext with asymmetric public key. + + Returns: + Union[str, bytes]: Encrypted ciphertext as string or bytes. + """ + + if isinstance(plaintext, str): + plaintext = plaintext.encode() + + _ciphertext: Union[str, bytes] + try: + _message = "Encrypting plaintext with asymmetric public key..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _ciphertext: bytes = public_key.encrypt( + plaintext=plaintext, + padding=padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + + _message = "Successfully encrypted plaintext with asymmetric public key." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + except Exception: + _message = "Failed to encrypt plaintext with asymmetric public key!" + if warn_mode == WarnEnum.ALWAYS: + logger.error(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + raise + + if base64_encode: + _ciphertext = base64.b64encode(_ciphertext) + + if as_str: + _ciphertext = _ciphertext.decode() + + return _ciphertext + + +@validate_call(config={"arbitrary_types_allowed": True}) +def decrypt_with_private_key( + ciphertext: Union[str, bytes], + private_key: PrivateKeyTypes, + base64_decode: bool = False, + as_str: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> Union[str, bytes]: + """Decrypt ciphertext with private key. + + Args: + ciphertext (Union[str, bytes], required): Ciphertext to decrypt. + private_key (PrivateKeyTypes , required): Private key. + base64_decode (bool , optional): Decode ciphertext with base64. Defaults to False. + as_str (bool , optional): Return plaintext as string or bytes. Defaults to False. + warn_mode (WarnEnum , optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + Exception: If failed to decrypt ciphertext with asymmetric private key for any reason. + + Returns: + Union[str, bytes]: Decrypted plaintext as string or bytes. + """ + + if isinstance(ciphertext, str): + ciphertext = ciphertext.encode() + + if base64_decode: + ciphertext = base64.b64decode(ciphertext) + + _plaintext: Union[str, bytes] + try: + _message = "Decrypting ciphertext with asymmetric private key..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _plaintext: bytes = private_key.decrypt( + ciphertext=ciphertext, + padding=padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA256()), + algorithm=hashes.SHA256(), + label=None, + ), + ) + + _message = "Successfully decrypted ciphertext with asymmetric private key." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + except Exception: + _message = "Failed to decrypt ciphertext with asymmetric private key!" + if warn_mode == WarnEnum.ALWAYS: + logger.error(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + raise + + if as_str: + _plaintext = _plaintext.decode() + + return _plaintext + + +__all__ = [ + "gen_key_pair", + "async_create_keys", + "async_get_private_key", + "async_get_public_key", + "async_get_keys", + "create_keys", + "get_private_key", + "get_public_key", + "get_keys", + "encrypt_with_public_key", + "decrypt_with_private_key", +] diff --git a/src/bv_challenge/challenge/api/helpers/crypto/ssl.py b/src/bv_challenge/challenge/api/helpers/crypto/ssl.py new file mode 100644 index 0000000..c232a12 --- /dev/null +++ b/src/bv_challenge/challenge/api/helpers/crypto/ssl.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- + +import os +import errno +from datetime import timedelta +from typing import Union + +import aiofiles +import aiofiles.os +from pydantic import validate_call, BaseModel, Field +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey +from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes +from beans_logging import logger + +from api.core.constants import WarnEnum +from api.core import utils + +from . import asymmetric as asymmetric_helper + + +class X509AttrsPM(BaseModel): + C: str = Field(default="US", min_length=2, max_length=2) + ST: str = Field(default="Washington", min_length=2, max_length=256) + L: str = Field(default="Seattle", min_length=2, max_length=256) + O: str = Field(default="Organization", min_length=2, max_length=256) + OU: str = Field(default="Organization Unit", min_length=2, max_length=256) + CN: str = Field(default="localhost", min_length=2, max_length=256) + DNS: str = Field(default="localhost", min_length=2, max_length=256) + + +@validate_call +async def async_create_ssl_certs( + ssl_dir: str, + cert_fname: str, + key_fname: str, + key_size: int, + x509_attrs: X509AttrsPM = X509AttrsPM(), + force: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Async generate and create SSL key and cert files. + + Args: + ssl_dir (str , required): SSL directory path. + cert_fname (str , required): Certificate file name. + key_fname (str , required): Key file name. + key_size (int , required): Key size. + x509_attrs (X509AttrsPM, optional): X509 named attributes. Defaults to X509AttrsPM(). + force (bool , optional): Force to create SSL key and cert files. Defaults to False. + warn_mode (WarnEnum , optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + FileExistsError: When warning mode is set to ERROR and SSL key or cert files already exist. + OSError : If failed to create SSL key or cert files. + """ + + _key_path = os.path.join(ssl_dir, key_fname) + _cert_path = os.path.join(ssl_dir, cert_fname) + + if force: + await utils.async_remove_file(file_path=_key_path, warn_mode=warn_mode) + await utils.async_remove_file(file_path=_cert_path, warn_mode=warn_mode) + + if (await aiofiles.os.path.isfile(_key_path)) and ( + await aiofiles.os.path.isfile(_cert_path) + ): + logger.trace( + f"SSL key and cert files already exist: ['{_key_path}', '{_cert_path}']" + ) + return + + _meesage = f"Generating SSL key and cert files: ['{_key_path}', '{_cert_path}']..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_meesage) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_meesage) + + _private_key: Union[RSAPrivateKey, PrivateKeyTypes] + if await aiofiles.os.path.isfile(_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_key_path}' SSL key file already exists!") + + _private_key: PrivateKeyTypes = await asymmetric_helper.async_get_private_key( + private_key_path=_key_path + ) + else: + _private_key: RSAPrivateKey = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + + if await aiofiles.os.path.isfile(_cert_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_cert_path}' SSL cert file already exists!") + + await utils.async_remove_file(file_path=_cert_path, warn_mode=warn_mode) + + _subject = _issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, x509_attrs.C), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, x509_attrs.ST), + x509.NameAttribute(NameOID.LOCALITY_NAME, x509_attrs.L), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, x509_attrs.O), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, x509_attrs.OU), + x509.NameAttribute(NameOID.COMMON_NAME, x509_attrs.CN), + ] + ) + _cert = ( + x509.CertificateBuilder() + .subject_name(name=_subject) + .issuer_name(name=_issuer) + .public_key(key=_private_key.public_key()) + .serial_number(number=x509.random_serial_number()) + .not_valid_before(time=utils.now_utc_dt()) + .not_valid_after(time=utils.now_utc_dt() + timedelta(days=365)) + .add_extension( + extval=x509.SubjectAlternativeName([x509.DNSName(x509_attrs.DNS)]), + critical=False, + ) + .sign(private_key=_private_key, algorithm=hashes.SHA256()) + ) + + await utils.async_create_dir(create_dir=ssl_dir, warn_mode=warn_mode) + + if not await aiofiles.os.path.isfile(_key_path): + try: + async with aiofiles.open(_key_path, "wb") as _key_file: + await _key_file.write( + _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_key_path}' SSL key file already exists!") + else: + logger.error(f"Failed to create '{_key_path}' SSL key file!") + raise + + if not await aiofiles.os.path.isfile(_cert_path): + try: + async with aiofiles.open(_cert_path, "wb") as _cert_file: + await _cert_file.write(_cert.public_bytes(serialization.Encoding.PEM)) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_cert_path}' SSL cert file already exists!") + else: + logger.error(f"Failed to create '{_cert_path}' SSL cert file!") + raise + + _message = f"Successfully generated SSL key and cert files: ['{_key_path}', '{_cert_path}']" + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + return + + +@validate_call +def create_ssl_certs( + ssl_dir: str, + key_fname: str, + cert_fname: str, + key_size: int, + x509_attrs: X509AttrsPM = X509AttrsPM(), + force: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> None: + """Generate and create SSL key and cert files. + + Args: + ssl_dir (str , required): SSL directory path. + key_fname (str , required): Key file name. + cert_fname (str , required): Certificate file name. + key_size (int , required): Key size. + x509_attrs (X509AttrsPM, optional): X509 named attributes. Defaults to X509AttrsPM(). + force (bool , optional): Force to create SSL key and cert files. Defaults to False. + warn_mode (WarnEnum , optional): Warning mode. Defaults to WarnEnum.DEBUG. + + Raises: + FileExistsError: When warning mode is set to ERROR and SSL key or cert files already exist. + OSError : If failed to create SSL key or cert files. + """ + + _key_path = os.path.join(ssl_dir, key_fname) + _cert_path = os.path.join(ssl_dir, cert_fname) + + if force: + utils.remove_file(file_path=_key_path, warn_mode=warn_mode) + utils.remove_file(file_path=_cert_path, warn_mode=warn_mode) + + if os.path.isfile(_key_path) and os.path.isfile(_cert_path): + logger.trace( + f"SSL key and cert files already exist: ['{_key_path}', '{_cert_path}']" + ) + return + + _message = f"Generating SSL key and cert files: ['{_key_path}', '{_cert_path}']..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _private_key: Union[RSAPrivateKey, PrivateKeyTypes] + if os.path.isfile(_key_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_key_path}' SSL key file already exists!") + + _private_key: PrivateKeyTypes = asymmetric_helper.get_private_key( + private_key_path=_key_path + ) + else: + _private_key: RSAPrivateKey = rsa.generate_private_key( + public_exponent=65537, key_size=key_size + ) + + if os.path.isfile(_cert_path): + if warn_mode == WarnEnum.ERROR: + raise FileExistsError(f"'{_cert_path}' SSL cert file already exists!") + + utils.remove_file(file_path=_cert_path, warn_mode=warn_mode) + + _subject = _issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, x509_attrs.C), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, x509_attrs.ST), + x509.NameAttribute(NameOID.LOCALITY_NAME, x509_attrs.L), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, x509_attrs.O), + x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, x509_attrs.OU), + x509.NameAttribute(NameOID.COMMON_NAME, x509_attrs.CN), + ] + ) + _cert = ( + x509.CertificateBuilder() + .subject_name(name=_subject) + .issuer_name(name=_issuer) + .public_key(key=_private_key.public_key()) + .serial_number(number=x509.random_serial_number()) + .not_valid_before(time=utils.now_utc_dt()) + .not_valid_after(time=utils.now_utc_dt() + timedelta(days=365)) + .add_extension( + extval=x509.SubjectAlternativeName([x509.DNSName(x509_attrs.DNS)]), + critical=False, + ) + .sign(private_key=_private_key, algorithm=hashes.SHA256()) + ) + + utils.create_dir(create_dir=ssl_dir, warn_mode=warn_mode) + + if not os.path.isfile(_key_path): + try: + with open(_key_path, "wb") as _key_file: + _key_file.write( + _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_key_path}' SSL key file already exists!") + else: + logger.error(f"Failed to create '{_key_path}' SSL key file!") + raise + + if not os.path.isfile(_cert_path): + try: + with open(_cert_path, "wb") as _cert_file: + _cert_file.write(_cert.public_bytes(serialization.Encoding.PEM)) + + except OSError as err: + if (err.errno == errno.EEXIST) and (warn_mode == WarnEnum.DEBUG): + logger.debug(f"'{_cert_path}' SSL cert file already exists!") + else: + logger.error(f"Failed to create '{_cert_path}' SSL cert file!") + raise + + _message = f"Successfully generated SSL key and cert files: ['{_key_path}', '{_cert_path}']" + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + return + + +__all__ = [ + "async_create_ssl_certs", + "create_ssl_certs", +] diff --git a/src/bv_challenge/challenge/api/helpers/crypto/symmetric.py b/src/bv_challenge/challenge/api/helpers/crypto/symmetric.py new file mode 100644 index 0000000..0209f61 --- /dev/null +++ b/src/bv_challenge/challenge/api/helpers/crypto/symmetric.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +import base64 +from typing import Union + +from cryptography.hazmat.primitives import ciphers +from cryptography.hazmat.primitives.ciphers import algorithms, modes +from cryptography.hazmat.primitives import padding +from pydantic import validate_call +from beans_logging import logger + +from api.core.constants import WarnEnum + + +@validate_call(config={"arbitrary_types_allowed": True}) +def decrypt_aes_cbc( + ciphertext: Union[str, bytes], + key: bytes, + iv: bytes, + base64_decode: bool = False, + as_str: bool = False, + warn_mode: WarnEnum = WarnEnum.DEBUG, +) -> Union[str, bytes]: + """Decrypts a ciphertext using AES-CBC key and iv. + + Args: + ciphertext (Union[str, bytes], required): The ciphertext to decrypt. + key (bytes , required): The key to use for decryption. + iv (bytes , required): The initialization vector to use for decryption. + base64_decode (bool , optional): Whether to decode the ciphertext from base64. Defaults to False. + as_str (bool , optional): Whether to return the plaintext as a string or bytes. Defaults to False. + warn_mode (WarnEnum , optional): The warning mode to use. Defaults to WarnEnum.DEBUG. + + Raises: + Exception: If failed to decrypt ciphertext using AES-CBC key and iv for any reason. + + Returns: + Union[str, bytes]: The decrypted plaintext as a string or bytes. + """ + + if isinstance(ciphertext, str): + ciphertext = ciphertext.encode() + + if base64_decode: + ciphertext = base64.b64decode(ciphertext) + + _plaintext: Union[str, bytes] + try: + _message = "Decrypting ciphertext using AES-CBC key and iv..." + if warn_mode == WarnEnum.ALWAYS: + logger.info(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + _cipher = ciphers.Cipher( + algorithm=algorithms.AES(key=key), mode=modes.CBC(initialization_vector=iv) + ) + _decryptor = _cipher.decryptor() + _padded_plaintext = _decryptor.update(data=ciphertext) + _decryptor.finalize() + + _unpadder = padding.PKCS7(block_size=algorithms.AES.block_size).unpadder() + _plaintext = _unpadder.update(_padded_plaintext) + _unpadder.finalize() + + _message = "Successfully decrypted ciphertext using AES-CBC key and iv." + if warn_mode == WarnEnum.ALWAYS: + logger.success(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + except Exception: + _message = "Failed to decrypt ciphertext using AES-CBC key and iv!" + if warn_mode == WarnEnum.ALWAYS: + logger.error(_message) + elif warn_mode == WarnEnum.DEBUG: + logger.debug(_message) + + raise + + if as_str: + _plaintext = _plaintext.decode() + + return _plaintext + + +__all__ = [ + "decrypt_aes_cbc", +] diff --git a/src/bv_challenge/challenge/api/lifespan.py b/src/bv_challenge/challenge/api/lifespan.py index 5caee29..4ad25f7 100644 --- a/src/bv_challenge/challenge/api/lifespan.py +++ b/src/bv_challenge/challenge/api/lifespan.py @@ -1,27 +1,23 @@ +# -*- coding: utf-8 -*- + import os -from collections.abc import AsyncGenerator +from typing import AsyncGenerator from contextlib import asynccontextmanager from fastapi import FastAPI -from potato_util.io import async_create_dir -from potato_util.crypto import asymmetric as asymmetric_utils -from potato_util.crypto import ssl as ssl_utils - -from api.__version__ import __version__ +from api.core import utils from api.config import config +from api.helpers.crypto import asymmetric as asymmetric_helper +from api.helpers.crypto import ssl as ssl_helper from api.logger import logger -def _check_ssl_certs() -> None: - """Check if SSL certificates exist when SSL is enabled or set to be generated. - - Raises: - SystemExit: If SSL certificates are missing or cannot be created. - """ +def pre_init() -> None: + """Pre-initialization tasks before creating FastAPI application.""" if config.api.security.ssl.generate: - ssl_utils.create_ssl_certs( + ssl_helper.create_ssl_certs( ssl_dir=config.api.paths.ssl_dir, key_fname=config.api.security.ssl.key_fname, cert_fname=config.api.security.ssl.cert_fname, @@ -30,31 +26,20 @@ def _check_ssl_certs() -> None: ) if config.api.security.ssl.enabled: - _ssl_keyfile_path = os.path.join( + _ssl_keyfile = os.path.join( config.api.paths.ssl_dir, config.api.security.ssl.key_fname ) - _ssl_certfile_path = os.path.join( + _ssl_certfile = os.path.join( config.api.paths.ssl_dir, config.api.security.ssl.cert_fname ) - if (not os.path.isfile(_ssl_keyfile_path)) or ( - not os.path.isfile(_ssl_certfile_path) - ): + if (not os.path.isfile(_ssl_keyfile)) or (not os.path.isfile(_ssl_certfile)): logger.error("SSL key or certificate file not found!") raise SystemExit(1) return -def pre_init() -> None: - """Pre-initialization tasks before creating FastAPI application.""" - - _check_ssl_certs() - # Add more pre-initialization tasks here... - - return - - async def _async_create_dirs() -> None: """Create directories before starting FastAPI application. @@ -63,8 +48,8 @@ async def _async_create_dirs() -> None: """ try: - await async_create_dir(config.api.paths.data_dir) - # Add directories that need to be created here... + await utils.async_create_dir(config.api.paths.data_dir) + ## Add directories needs to be created here... except Exception: logger.exception("Failed to create directories:") raise SystemExit(1) @@ -84,16 +69,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: logger.info("Preparing to startup...") # await _async_create_dirs() if config.api.security.asymmetric.generate: - await asymmetric_utils.async_create_keys( + await asymmetric_helper.async_create_keys( asymmetric_keys_dir=config.api.paths.asymmetric_keys_dir, key_size=config.api.security.asymmetric.key_size, private_key_fname=config.api.security.asymmetric.private_key_fname, public_key_fname=config.api.security.asymmetric.public_key_fname, ) - # Add startup code here... + ## Add startup code here... logger.success("Finished preparation to startup.") - logger.opt(colors=True).info(f"Version: {__version__}") + logger.opt(colors=True).info(f"Version: {config.version}") logger.opt(colors=True).info(f"API version: {config.api.version}") logger.opt(colors=True).info(f"API prefix: {config.api.prefix}") logger.opt(colors=True).info( @@ -103,7 +88,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: yield logger.info("Praparing to shutdown...") - # Add shutdown code here... + ## Add shutdown code here... logger.success("Finished preparation to shutdown.") diff --git a/src/bv_challenge/challenge/api/logger.py b/src/bv_challenge/challenge/api/logger.py index 46da0bd..d2b046b 100644 --- a/src/bv_challenge/challenge/api/logger.py +++ b/src/bv_challenge/challenge/api/logger.py @@ -1,5 +1,124 @@ -from beans_logging_fastapi import logger +# -*- coding: utf-8 -*- + +from pydantic import validate_call +from fastapi.concurrency import run_in_threadpool + +from beans_logging import Logger, LoggerLoader +from beans_logging_fastapi import ( + add_http_file_handler, + add_http_file_json_handler, + http_file_format, +) + +from api.core.constants import WarnEnum +from api.config import config + + +logger_loader = LoggerLoader(config=config.logger, auto_config_file=False) +logger: Logger = logger_loader.load() + + +def _http_file_format(record: dict) -> str: + _format = http_file_format( + record=record, + msg_format=config.logger.extra.http_file_format, + tz=config.logger.extra.http_file_tz, + ) + return _format + + +if config.logger.extra.http_file_enabled: + add_http_file_handler( + logger_loader=logger_loader, + log_path=config.logger.extra.http_log_path, + err_path=config.logger.extra.http_err_path, + formatter=_http_file_format, + ) + +if config.logger.extra.http_json_enabled: + add_http_file_json_handler( + logger_loader=logger_loader, + log_path=config.logger.extra.http_json_path, + err_path=config.logger.extra.http_json_err_path, + ) + + +@validate_call +def log_mode( + message: str, level: str = "INFO", warn_mode: WarnEnum = WarnEnum.ALWAYS +) -> None: + """Log message with level and warn mode. + + Args: + message (str, reqiured): Message to log. + level (LogLevelEnum, optional): Log level when warn mode is `WarnEnum.ALWAYS`. Defaults to "INFO". + warn_mode (WarnEnum, optional): Warn mode to use. Defaults to `WarnEnum.ALWAYS`. + + Raises: + ValueError: If `level` is not a valid log level. + """ + + level = level.upper() + if warn_mode == WarnEnum.ALWAYS: + if level == "INFO": + logger.info(message) + elif level == "SUCCESS": + logger.success(message) + elif level == "WARNING": + logger.warning(message) + elif level == "ERROR": + logger.error(message) + elif level == "CRITICAL": + logger.critical(message) + elif level == "TRACE": + logger.trace(message) + else: + raise ValueError(f"Unknown log level: '{level}'") + + elif warn_mode == WarnEnum.DEBUG: + logger.debug(message) + + return + + +@validate_call +async def async_log_mode( + message: str, level: str = "INFO", warn_mode: WarnEnum = WarnEnum.ALWAYS +) -> None: + """Log message with level and warn mode in async mode. + + Args: + message (str , required): Message to log. + level (str , optional): Log level when warn mode is `WarnEnum.ALWAYS`. Defaults to "INFO". + warn_mode (WarnEnum, optional): Warn mode to use. Defaults to `WarnEnum.ALWAYS`. + """ + + level = level.upper() + if warn_mode == WarnEnum.ALWAYS: + if level == "INFO": + await run_in_threadpool(logger.info, message) + elif level == "SUCCESS": + await run_in_threadpool(logger.success, message) + elif level == "WARNING": + await run_in_threadpool(logger.warning, message) + elif level == "ERROR": + await run_in_threadpool(logger.error, message) + elif level == "CRITICAL": + await run_in_threadpool(logger.critical, message) + elif level == "TRACE": + await run_in_threadpool(logger.trace, message) + else: + raise ValueError(f"Unknown log level: '{level}'") + + elif warn_mode == WarnEnum.DEBUG: + await run_in_threadpool(logger.debug, message) + + return + __all__ = [ + "logger_loader", "logger", + "log_mode", + "async_log_mode", ] diff --git a/src/bv_challenge/challenge/api/main.py b/src/bv_challenge/challenge/api/main.py index e748463..4b2e097 100644 --- a/src/bv_challenge/challenge/api/main.py +++ b/src/bv_challenge/challenge/api/main.py @@ -1,23 +1,5 @@ -# Third-party libraries -from dotenv import load_dotenv -from fastapi import FastAPI +# -*- coding: utf-8 -*- -load_dotenv(override=True) +from .__main__ import app, main -# Internal modules -from api.bootstrap import create_app, run_server # noqa: E402 - -app: FastAPI = create_app() - - -def main() -> None: - """Main function.""" - - run_server(app="api.main:app") - return - - -__all__ = [ - "app", - "main", -] +__all__ = ["app", "main"] diff --git a/src/bv_challenge/challenge/api/middleware.py b/src/bv_challenge/challenge/api/middleware.py index 1db2285..c238873 100644 --- a/src/bv_challenge/challenge/api/middleware.py +++ b/src/bv_challenge/challenge/api/middleware.py @@ -1,9 +1,17 @@ +# -*- coding: utf-8 -*- + from pydantic import validate_call from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware +from beans_logging_fastapi import ( + HttpAccessLogMiddleware, + RequestHTTPInfoMiddleware, + ResponseHTTPInfoMiddleware, +) + from api.config import config from api.core.middlewares import ProcessTimeMiddleware, RequestIdMiddleware @@ -16,8 +24,19 @@ def add_middlewares(app: FastAPI) -> None: app (FastAPI): FastAPI app instance. """ - # Add more middlewares here... - app.add_middleware(GZipMiddleware, **config.api.gzip.model_dump()) + ## Add more middlewares here... + app.add_middleware(ResponseHTTPInfoMiddleware) + app.add_middleware( + HttpAccessLogMiddleware, + debug_format=config.logger.extra.http_std_debug_format, + msg_format=config.logger.extra.http_std_msg_format, + ) + app.add_middleware( + RequestHTTPInfoMiddleware, + has_proxy_headers=config.api.behind_proxy, + has_cf_headers=config.api.behind_cf_proxy, + ) + # app.add_middleware(GZipMiddleware, minimum_size=config.api.gzip_min_size) app.add_middleware(CORSMiddleware, **config.api.security.cors.model_dump()) app.add_middleware( TrustedHostMiddleware, allowed_hosts=config.api.security.allowed_hosts diff --git a/src/bv_challenge/challenge/api/mount.py b/src/bv_challenge/challenge/api/mount.py index 1f1b005..209a265 100644 --- a/src/bv_challenge/challenge/api/mount.py +++ b/src/bv_challenge/challenge/api/mount.py @@ -1,9 +1,10 @@ -# import os +# -*- coding: utf-8 -*- + +import pathlib from pydantic import validate_call from fastapi import FastAPI - -# from fastapi.staticfiles import StaticFiles +from fastapi.staticfiles import StaticFiles @validate_call(config={"arbitrary_types_allowed": True}) @@ -14,9 +15,13 @@ def add_mounts(app: FastAPI) -> None: app (FastAPI): FastAPI app instance. """ - # app.mount("/static", StaticFiles(directory=os.path.join("api", "static")), name="static") - # Add mounts here + _src_dir = pathlib.Path(__file__).parent.parent.resolve() + app.mount( + path="/static", + app=StaticFiles(directory=str(_src_dir / "./templates/html/static")), + name="static", + ) return diff --git a/src/bv_challenge/challenge/api/router.py b/src/bv_challenge/challenge/api/router.py index c5e6560..8fd1a8a 100644 --- a/src/bv_challenge/challenge/api/router.py +++ b/src/bv_challenge/challenge/api/router.py @@ -1,9 +1,10 @@ +# -*- coding: utf-8 -*- + from pydantic import validate_call from fastapi import FastAPI, APIRouter from api.config import config from api.core.routers.utils import router as utils_router -from api.core.routers.default import router as default_router from api.endpoints.challenge.router import router as challenge_router @@ -16,14 +17,11 @@ def add_routers(app: FastAPI) -> None: """ _api_router = APIRouter(prefix=config.api.prefix) - _api_router.include_router(challenge_router) _api_router.include_router(utils_router) + _api_router.include_router(challenge_router) # Add more API routers here... - # Add admin API routers here... - app.include_router(_api_router) - app.include_router(default_router) return diff --git a/src/bv_challenge/challenge/api/static/css/.gitkeep b/src/bv_challenge/challenge/api/static/css/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/static/fonts/.gitkeep b/src/bv_challenge/challenge/api/static/fonts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/static/images/favicon.ico b/src/bv_challenge/challenge/api/static/images/favicon.ico deleted file mode 100644 index 9efd3bf..0000000 Binary files a/src/bv_challenge/challenge/api/static/images/favicon.ico and /dev/null differ diff --git a/src/bv_challenge/challenge/api/static/index.html b/src/bv_challenge/challenge/api/static/index.html deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/static/js/.gitkeep b/src/bv_challenge/challenge/api/static/js/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/static/media/.gitkeep b/src/bv_challenge/challenge/api/static/media/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/templates/html/.gitkeep b/src/bv_challenge/challenge/api/templates/html/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/api/templates/mail/.gitkeep b/src/bv_challenge/challenge/api/templates/mail/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/bv_challenge/challenge/requirements.txt b/src/bv_challenge/challenge/requirements.txt index 4870bad..929f90f 100644 --- a/src/bv_challenge/challenge/requirements.txt +++ b/src/bv_challenge/challenge/requirements.txt @@ -1,5 +1,16 @@ -certifi>=2024.2.2,<2030.0.0 -anyio>=4.3.0,<5.0.0 -potato-util[crypto,async]~=0.5.3 -beans-logging-fastapi~=6.0.4 -fastapi[all]~=0.135.1 +pycparser>=2.22,<3.0.0 +certifi>=2024.8.30,<2030.0.0 +Mako>=1.3.6,<2.0.0 +argon2-cffi-bindings>=21.2.0,<22.0.0 +aioshutil~=1.5 +aiofiles~=24.1.0 +PyJWT~=2.10.1 +cryptography>=43.0.0,<50.0.0 +argon2-cffi~=23.1.0 +beans-logging-fastapi~=1.1.1 +onion-config[pydantic-settings]~=5.1.1 +aiohttp~=3.10.2 +fastapi[all]~=0.110.1 +docker~=7.1.0 +./requirements/vault_unlock-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl +./requirements/rt_bv_score-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl diff --git a/src/bv_challenge/challenge/requirements/rt_bv_score-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl b/src/bv_challenge/challenge/requirements/rt_bv_score-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl new file mode 100644 index 0000000..d3803b5 Binary files /dev/null and b/src/bv_challenge/challenge/requirements/rt_bv_score-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl differ diff --git a/src/bv_challenge/challenge/requirements/vault_unlock-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl b/src/bv_challenge/challenge/requirements/vault_unlock-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl new file mode 100644 index 0000000..4a4fdc4 Binary files /dev/null and b/src/bv_challenge/challenge/requirements/vault_unlock-0.1.0-cp310-abi3-manylinux_2_34_x86_64.whl differ diff --git a/src/bv_challenge/challenge/templates/html/asset-manifest.json b/src/bv_challenge/challenge/templates/html/asset-manifest.json new file mode 100644 index 0000000..e46061f --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/asset-manifest.json @@ -0,0 +1,11 @@ +{ + "files": { + "collector.js": "/static/js/collector.js", + "sdk.js": "/static/js/sdk.js", + "index.html": "/index.html" + }, + "entrypoints": [ + "static/js/collector.js", + "static/js/sdk.js" + ] +} diff --git a/src/bv_challenge/challenge/templates/html/index.html b/src/bv_challenge/challenge/templates/html/index.html new file mode 100644 index 0000000..3276b1c --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/index.html @@ -0,0 +1,84 @@ + + + + + + Browser verification + + + + +
+

Browser verification

+

Checking browser environment...

+
Checking…
+
+ + + + + + diff --git a/src/bv_challenge/challenge/templates/html/robots.txt b/src/bv_challenge/challenge/templates/html/robots.txt new file mode 100644 index 0000000..e9e57dc --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/src/bv_challenge/challenge/templates/html/static/css/main.963c38ad.css b/src/bv_challenge/challenge/templates/html/static/css/main.963c38ad.css new file mode 100644 index 0000000..2fa2fb6 --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/static/css/main.963c38ad.css @@ -0,0 +1 @@ +.login-container{background-color:#fff;border-radius:8px;box-shadow:0 4px 8px #0000001a;max-width:400px;padding:20px;text-align:center;width:100%}.login-form h2{color:#333;margin-bottom:10px}.login-form p{color:#666;font-size:.9rem;margin-bottom:20px}.form-group{margin-bottom:15px}.form-group input{border:1px solid #ccc;border-radius:5px;font-size:1rem;padding:10px;width:360px}button#login-button{background-color:#007bff;border:none;border-radius:5px;color:#fff;cursor:pointer;font-size:1rem;padding:10px;width:100%}button#login-button:hover{background-color:#0056b3}.message.error{color:red;font-size:.9rem;margin-top:10px}.message.success{color:green;font-size:.9rem;margin-top:10px}body,html{height:100%;margin:0;overflow-x:hidden}.app{align-items:center;background-color:#f9f9f9;box-sizing:border-box;display:flex;flex-direction:column;font-family:Arial,sans-serif;justify-content:flex-start;margin:0;min-height:150vh;padding:20px;position:relative}.app-header{margin-bottom:20px;text-align:center}.app-header h1{color:#333;font-size:1.8rem;margin-bottom:10px}.app-header p{color:#555;font-size:1rem}.app-header ol{margin:10px 0;max-width:600px;padding-left:20px;text-align:left}.content{align-items:center;display:flex;flex-direction:column;gap:20px;justify-content:center;max-width:400px;text-align:center;width:100%}#actions-list-display{box-shadow:-2px 0 5px #0000001a;box-sizing:border-box;height:100vh;overflow-y:auto;padding:20px;position:fixed;right:0;top:0;width:15%}.action-item{background-color:#fff;border:1px solid #ddd;border-radius:4px;color:red;margin:5px 0;padding:10px;transition:all .3s ease}.action-item.current{background-color:#e3f2fd;border-color:#2196f3;box-shadow:0 0 5px #2196f34d}.action-item.completed{background-color:#e8f5e9;border-color:#4caf50;color:green}.action-item.pending{opacity:.7}.action-item h3{color:#333;font-size:14px;margin:0 0 5px}.action-coordinates{color:#666;font-size:12px}.check-mark{color:#4caf50;margin-left:8px}.current-marker{color:#2196f3;margin-left:8px}.progress{background-color:#f5f5f5;border-radius:4px;font-weight:700;margin-top:20px;padding:10px;text-align:center}.login-form-container{flex-direction:column;min-height:100vh;padding-bottom:40px;width:70%}.login-form-container,.submit-button-container{align-items:center;display:flex;justify-content:center}.submit-button-container{margin-top:3000px;padding:40px 0;width:90%}.submit-button{background-color:#007bff;border:none;border-radius:5px;box-shadow:0 2px 4px #0000001a;color:#fff;cursor:pointer;font-size:1.1rem;padding:15px 30px;transition:background-color .3s ease}.submit-button:hover{background-color:#0056b3;box-shadow:0 4px 8px #00000026}.download-button-container{margin-top:20px;width:auto}.download-button-container button{background-color:#28a745;border:none;border-radius:5px;color:#fff;cursor:pointer;font-size:1rem;padding:10px 20px;transition:background-color .3s ease} \ No newline at end of file diff --git a/src/bv_challenge/challenge/templates/html/static/js/collector.js b/src/bv_challenge/challenge/templates/html/static/js/collector.js new file mode 100644 index 0000000..05f8ee1 --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/static/js/collector.js @@ -0,0 +1,151 @@ +/** + * Bot Virus — browser/runtime integrity collector (pure, DOM-free). + * + * HBv6 no longer scores behavior. This module only *accumulates* raw, + * non-behavioral browser/runtime/session-integrity observations into a plain + * payload object. It contains NO scoring logic and no detection heuristics — + * those live server-side in the private detector. Keeping this layer dumb makes + * it trivially unit-testable in Node and keeps the public frontend free of + * anything a miner could game. + * + * Every field stored here is a *raw observation* (an environment string, a + * capability boolean, a fingerprint summary, a display dimension) — never a + * formula. No mouse/keyboard/scroll/click/task signal is collected. + * + * The advanced sections (automation, runtimeIntegrity, fingerprint, + * apiAvailability, navigator, display, correlation, sessionBinding) are written + * once each (first-write-wins). The legacy raw behavior series are retained as + * EMPTY lists for backward compatibility with the public Layer 1 shape gate; + * the page and SDK never populate them. + * + * UMD wrapper: usable as a classic browser script (window.BVCollector) and as a + * CommonJS module (require) so the same file is exercised by the Node tests. + */ +(function (root, factory) { + if (typeof module === "object" && module.exports) { + module.exports = factory(); + } else { + root.BVCollector = factory(); + } +})(typeof self !== "undefined" ? self : this, function () { + "use strict"; + + // Advanced non-behavioral sections the collector accepts. Unknown section + // names are ignored so the frontend can never inject arbitrary keys. + var SECTION_KEYS = { + automation: true, + runtimeIntegrity: true, + fingerprint: true, + apiAvailability: true, + navigator: true, + display: true, + correlation: true, + sessionBinding: true + }; + + // Legacy raw behavior series. Retained as empty lists only so older payloads + // and the public shape gate stay happy — never populated in HBv6. + var LEGACY_SERIES = [ + "movements", + "clicks", + "mouseDowns", + "mouseUps", + "keydowns", + "keyups", + "scroll" + ]; + + function createCollector(options) { + var opts = options || {}; + var now = + typeof opts.now === "function" + ? opts.now + : function () { + return Date.now(); + }; + + var data = { + schemaVersion: opts.schemaVersion || null, + sessionId: opts.sessionId || null, + startedAt: now(), + endedAt: null, + + // --- raw browser/runtime integrity sections (first-write-wins) -------- + browserInfo: null, + automation: null, + runtimeIntegrity: null, + fingerprint: null, + apiAvailability: null, + navigator: null, + display: null, + correlation: null, + sessionBinding: null, + + // Page load timing (navigation timing API) — non-behavioral. + pageTimings: { pageLoadMs: null } + }; + + // Legacy empty series (back-compat with the public Layer 1 shape gate). + for (var i = 0; i < LEGACY_SERIES.length; i++) { + data[LEGACY_SERIES[i]] = []; + } + + /** + * Store the raw browser/environment snapshot. First write wins so a single + * session reports one consistent environment. + */ + function setBrowserInfo(info) { + if (data.browserInfo === null && info && typeof info === "object") { + data.browserInfo = info; + } + } + + /** + * Store one advanced non-behavioral section (first-write-wins). Unknown + * section names are ignored. + */ + function setSection(name, value) { + if ( + SECTION_KEYS[name] && + data[name] === null && + value && + typeof value === "object" + ) { + data[name] = value; + } + } + + /** Set a single pageTimings field once (first write wins). */ + function setPageTiming(key, value) { + if ( + Object.prototype.hasOwnProperty.call(data.pageTimings, key) && + data.pageTimings[key] === null && + typeof value === "number" + ) { + data.pageTimings[key] = value; + } + } + + function finalize() { + if (data.endedAt === null) { + data.endedAt = now(); + } + return data; + } + + function toJSON() { + return data; + } + + return { + data: data, + setBrowserInfo: setBrowserInfo, + setSection: setSection, + setPageTiming: setPageTiming, + finalize: finalize, + toJSON: toJSON + }; + } + + return { createCollector: createCollector, SECTION_KEYS: SECTION_KEYS }; +}); diff --git a/src/bv_challenge/challenge/templates/html/static/js/main.b9668575.js b/src/bv_challenge/challenge/templates/html/static/js/main.b9668575.js new file mode 100644 index 0000000..e58ad8a --- /dev/null +++ b/src/bv_challenge/challenge/templates/html/static/js/main.b9668575.js @@ -0,0 +1,2 @@ +/*! For license information please see main.b9668575.js.LICENSE.txt */ +(()=>{var A={49:(A,I)=>{"use strict";var g=Symbol.for("react.element"),B=Symbol.for("react.portal"),Q=Symbol.for("react.fragment"),C=Symbol.for("react.strict_mode"),E=Symbol.for("react.profiler"),i=Symbol.for("react.provider"),o=Symbol.for("react.context"),D=Symbol.for("react.forward_ref"),a=Symbol.for("react.suspense"),e=Symbol.for("react.memo"),t=Symbol.for("react.lazy"),n=Symbol.iterator;var w={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},c=Object.assign,G={};function r(A,I,g){this.props=A,this.context=I,this.refs=G,this.updater=g||w}function s(){}function N(A,I,g){this.props=A,this.context=I,this.refs=G,this.updater=g||w}r.prototype.isReactComponent={},r.prototype.setState=function(A,I){if("object"!==typeof A&&"function"!==typeof A&&null!=A)throw Error("setState(...): takes an object of state variables to update or a function which returns an object of state variables.");this.updater.enqueueSetState(this,A,I,"setState")},r.prototype.forceUpdate=function(A){this.updater.enqueueForceUpdate(this,A,"forceUpdate")},s.prototype=r.prototype;var k=N.prototype=new s;k.constructor=N,c(k,r.prototype),k.isPureReactComponent=!0;var M=Array.isArray,y=Object.prototype.hasOwnProperty,h={current:null},l={key:!0,ref:!0,__self:!0,__source:!0};function F(A,I,B){var Q,C={},E=null,i=null;if(null!=I)for(Q in void 0!==I.ref&&(i=I.ref),void 0!==I.key&&(E=""+I.key),I)y.call(I,Q)&&!l.hasOwnProperty(Q)&&(C[Q]=I[Q]);var o=arguments.length-2;if(1===o)C.children=B;else if(1{"use strict";!function A(){if("undefined"!==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"===typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(A)}catch(I){console.error(I)}}(),A.exports=g(345)},340:(A,I,g)=>{"use strict";A.exports=g(761)},345:(A,I,g)=>{"use strict";var B=g(950),Q=g(340);function C(A){for(var I="https://reactjs.org/docs/error-decoder.html?invariant="+A,g=1;g