feat: OpenTelemetry traces/metrics, JSON logs, job locks

.dev/docker-compose.yaml

@@ -13,4 +13,14 @@ services:
     image: redis:7-alpine
     ports:
       - "127.0.0.1:6379:6379"
-    restart: unless-stopped
+    restart: unless-stopped
+
+  # Optional: docker compose -f .dev/docker-compose.yaml --profile telemetry up -d
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.161.0
+    command: ["--config=/etc/otelcol/config.yaml"]
+    volumes:
+      - ./otel-collector.yaml:/etc/otelcol/config.yaml:ro
+    ports:
+      - "127.0.0.1:4318:4318"
+    profiles: [telemetry]

.dev/otel-collector.yaml

@@ -0,0 +1,20 @@
+# Local OpenTelemetry collector: prints what the app exports.
+# Point the app at it with OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318.
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug]

.gitattributes

@@ -0,0 +1,2 @@
+# LF everywhere, on every OS (matches ruff's line-ending = "lf").
+* text=auto eol=lf

.vscode/launch.json

@@ -1,23 +1,23 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "App Debug",
-            "type": "debugpy",
-            "request": "launch",
-            "module": "uvicorn",
-            "args": [
-                "app.main:app",
-                "--reload"
-            ],
-            "jinja": false,
-            "justMyCode": false,
-            "env": {
-                "WCS_ADMIN_TOKEN": "wcsadmin"
-            },
-        }
-    ]
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "App Debug",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "app.main:app",
+                "--reload"
+            ],
+            "jinja": false,
+            "justMyCode": false,
+            "env": {
+                "WCS_ADMIN_TOKEN": "wcsadmin"
+            },
+        }
+    ]
 }

Dockerfile

@@ -1,43 +1,46 @@
-FROM bufbuild/buf:1.66.0 AS buf_installer
-
-# Use a Python image with uv pre-installed
-FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
-
-# Install the project into `/app`
-WORKDIR /app
-COPY . /app
-
-# Copy the buf binary from the buf_installer stage and make it executable
-COPY --from=buf_installer /usr/local/bin/buf /usr/local/bin/buf
-
-RUN chmod +x /usr/local/bin/buf
-RUN buf generate --template buf.gen.yaml
-
-# Enable bytecode compilation
-ENV UV_COMPILE_BYTECODE=1
-
-# Copy from the cache instead of linking since it's a mounted volume
-ENV UV_LINK_MODE=copy
-
-# Install the project's dependencies using the lockfile and settings
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-dev
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --locked --no-dev
-
-# Place executables in the environment at the front of the path
-ENV PATH="/app/.venv/bin:$PATH"
-
-RUN addgroup --system --gid 1000 wynnsource && adduser --system --uid 1000 --ingroup wynnsource wynnsource
-RUN chown -R 1000:1000 /app
-
-ENV UV_NO_CACHE=1
-
-USER 1000
-
-# Run the FastAPI application
-EXPOSE 8000
-CMD fastapi run --forwarded-allow-ips="127.0.0.1,10.0.0.0/8,[::1]"
+FROM bufbuild/buf:1.66.0 AS buf_installer
+
+# Use a Python image with uv pre-installed
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+# Install the project into `/app`
+WORKDIR /app
+COPY . /app
+
+# Copy the buf binary from the buf_installer stage and make it executable
+COPY --from=buf_installer /usr/local/bin/buf /usr/local/bin/buf
+
+RUN chmod +x /usr/local/bin/buf
+RUN buf generate --template buf.gen.yaml
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --locked --no-install-project --no-dev
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-dev
+
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
+
+RUN addgroup --system --gid 1000 wynnsource && adduser --system --uid 1000 --ingroup wynnsource wynnsource
+RUN chown -R 1000:1000 /app
+
+ENV UV_NO_CACHE=1
+
+USER 1000
+
+# Run uvicorn directly in exec form: fastapi-cli re-attaches its own rich log
+# handlers after importing the app, bypassing loguru (and LOG_JSON), and the
+# shell form keeps uvicorn from receiving SIGTERM, which skips the telemetry
+# flush on shutdown.
+EXPOSE 8000
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--forwarded-allow-ips", "127.0.0.1,10.0.0.0/8,[::1]"]

README.md

@@ -25,6 +25,9 @@ The server component of [WynnSource](https://github.com/WynnSource) — a Wynncr
 git clone --recurse-submodules https://github.com/WynnSource/WynnSourceServer.git
 cd WynnSourceServer
 
+# Start PostgreSQL and Redis on localhost (add `--profile telemetry` for a local OTel collector)
+docker compose -f .dev/docker-compose.yaml up -d
+
 # Generate protobuf code
 buf generate --template buf.gen.yaml
 
@@ -38,6 +41,22 @@ uv run alembic upgrade head
 uv run fastapi dev
 ```
 
+### Windows
+
+Everything above works natively on Windows. Differences:
+
+- **Containers without Docker:** `wslc` (WSL containers) has no compose, start the services individually:
+
+  ```powershell
+  wslc run -d --name wcs-db -e POSTGRES_DB=wcs_db -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -p 127.0.0.1:5432:5432 postgres:15-alpine
+  wslc run -d --name wcs-redis -p 127.0.0.1:6379:6379 redis:7-alpine
+  # Optional local OTel collector (prints spans and metrics: wslc logs -f wcs-otelcol)
+  wslc run -d --name wcs-otelcol -p 127.0.0.1:4318:4318 -v "$PWD\.dev\otel-collector.yaml:/etc/otelcol/config.yaml" otel/opentelemetry-collector-contrib:0.161.0 --config=/etc/otelcol/config.yaml
+  ```
+
+- **Non-UTF-8 console code page** (e.g. GBK on Chinese Windows): `fastapi dev` crashes printing its emoji banner. Set `$env:PYTHONUTF8 = "1"`, or run `uv run uvicorn app.main:app --reload` instead.
+- **A `.venv` created from WSL or a container** contains Linux symlinks (`lib64`) that Windows `uv` cannot remove. Delete it from WSL (`rm -rf .venv`) and run `uv sync` again.
+
 ### Environment Variables
 
 | Variable | Description | Default |
@@ -51,7 +70,16 @@ uv run fastapi dev
 | `REDIS_PORT` | Redis port | `6379` |
 | `WCS_ADMIN_TOKEN` | Admin API token | None |
 | `LEVEL` | Log level | `DEBUG` |
+| `LOG_JSON` | Log one JSON object per line (with `trace_id`/`span_id`) instead of colored text | `false` |
 | `BETA_ALLOWED_VERSIONS` | Comma-separated allowed mod versions | `` |
+| `WCS_SENTRY_DSN` | Sentry DSN; errors and logs only, tagged with the OpenTelemetry trace id | None (disabled) |
+| `WCS_SENTRY_ENVIRONMENT` | Sentry environment | `local` |
+| `WCS_SENTRY_PROFILE_SESSION_SAMPLE_RATE` | Share of processes that run Sentry continuous profiling | `0` |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP/HTTP collector, e.g. `http://localhost:4318`; enables OpenTelemetry traces and metrics | None (disabled) |
+| `OTEL_RESOURCE_ATTRIBUTES` | Extra resource attributes, e.g. `service.instance.id=…,deployment.environment.name=dev`. Give every replica a distinct `service.instance.id` | — |
+| `K8S_POD_NAME` | Holder name for the cross-replica scheduler job locks | hostname |
+
+Other standard `OTEL_*` variables (`OTEL_TRACES_SAMPLER`, `OTEL_METRIC_EXPORT_INTERVAL`, …) are honored. The default sampler is `parentbased_always_on`: requests follow the caller's sampling decision, scheduler jobs are always traced.
 
 ## Deploying to Kubernetes
 

app/__init__.py

@@ -1 +1 @@
-# blank __init__.py file to mark the directory as a package for discoverability
+# blank __init__.py file to mark the directory as a package for discoverability

app/config/__init__.py

@@ -1,13 +1,15 @@
-from .admin import ADMIN_CONFIG as ADMIN_CONFIG
-from .db import DB_CONFIG as DB_CONFIG
-from .log import LOG_CONFIG as LOG_CONFIG
-from .sentry import SENTRY_CONFIG as SENTRY_CONFIG
-from .user import USER_CONFIG as USER_CONFIG
-
-__all__ = [
-    "ADMIN_CONFIG",
-    "DB_CONFIG",
-    "LOG_CONFIG",
-    "SENTRY_CONFIG",
-    "USER_CONFIG",
-]
+from .admin import ADMIN_CONFIG as ADMIN_CONFIG
+from .db import DB_CONFIG as DB_CONFIG
+from .log import LOG_CONFIG as LOG_CONFIG
+from .sentry import SENTRY_CONFIG as SENTRY_CONFIG
+from .telemetry import TELEMETRY_CONFIG as TELEMETRY_CONFIG
+from .user import USER_CONFIG as USER_CONFIG
+
+__all__ = [
+    "ADMIN_CONFIG",
+    "DB_CONFIG",
+    "LOG_CONFIG",
+    "SENTRY_CONFIG",
+    "TELEMETRY_CONFIG",
+    "USER_CONFIG",
+]

app/config/admin.py

@@ -1,17 +1,17 @@
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class AdminConfig(BaseSettings):
-    """
-    Configuration for the admin interface.
-    """
-
-    token: str | None = Field(alias="WCS_ADMIN_TOKEN", default=None)
-
-
-ADMIN_CONFIG = AdminConfig()  # Debugging line to check the environment variable
-
-__all__ = [
-    "ADMIN_CONFIG",
-]
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AdminConfig(BaseSettings):
+    """
+    Configuration for the admin interface.
+    """
+
+    token: str | None = Field(alias="WCS_ADMIN_TOKEN", default=None)
+
+
+ADMIN_CONFIG = AdminConfig()  # Debugging line to check the environment variable
+
+__all__ = [
+    "ADMIN_CONFIG",
+]

app/config/db.py

@@ -1,44 +1,44 @@
-from typing import Annotated
-
-from pydantic import Field, PostgresDsn, RedisDsn, computed_field
-from pydantic_settings import BaseSettings
-
-
-class DbConfig(BaseSettings):
-    postgres_host: Annotated[str, Field(alias="POSTGRES_HOST")] = "localhost"
-    postgres_port: Annotated[int, Field(alias="POSTGRES_PORT")] = 5432
-    postgres_user: Annotated[str, Field(alias="POSTGRES_USER")] = "postgres"
-    postgres_password: Annotated[str, Field(alias="POSTGRES_PASSWORD")] = "postgres"
-    postgres_db: Annotated[str, Field(alias="POSTGRES_DB")] = "wcs_db"
-
-    redis_host: Annotated[str, Field(alias="REDIS_HOST")] = "localhost"
-    redis_port: Annotated[int, Field(alias="REDIS_PORT")] = 6379
-
-    @computed_field
-    @property
-    def postgres_dsn(self) -> PostgresDsn:
-        return PostgresDsn.build(
-            scheme="postgresql+asyncpg",
-            username=self.postgres_user,
-            password=self.postgres_password,
-            host=self.postgres_host,
-            port=self.postgres_port,
-            path=self.postgres_db,
-        )
-
-    @computed_field
-    @property
-    def redis_dsn(self) -> RedisDsn:
-        return RedisDsn.build(
-            scheme="redis",
-            host=self.redis_host,
-            port=self.redis_port,
-            path="0",
-        )
-
-
-DB_CONFIG = DbConfig()
-
-__all__ = [
-    "DB_CONFIG",
-]
+from typing import Annotated
+
+from pydantic import Field, PostgresDsn, RedisDsn, computed_field
+from pydantic_settings import BaseSettings
+
+
+class DbConfig(BaseSettings):
+    postgres_host: Annotated[str, Field(alias="POSTGRES_HOST")] = "localhost"
+    postgres_port: Annotated[int, Field(alias="POSTGRES_PORT")] = 5432
+    postgres_user: Annotated[str, Field(alias="POSTGRES_USER")] = "postgres"
+    postgres_password: Annotated[str, Field(alias="POSTGRES_PASSWORD")] = "postgres"
+    postgres_db: Annotated[str, Field(alias="POSTGRES_DB")] = "wcs_db"
+
+    redis_host: Annotated[str, Field(alias="REDIS_HOST")] = "localhost"
+    redis_port: Annotated[int, Field(alias="REDIS_PORT")] = 6379
+
+    @computed_field
+    @property
+    def postgres_dsn(self) -> PostgresDsn:
+        return PostgresDsn.build(
+            scheme="postgresql+asyncpg",
+            username=self.postgres_user,
+            password=self.postgres_password,
+            host=self.postgres_host,
+            port=self.postgres_port,
+            path=self.postgres_db,
+        )
+
+    @computed_field
+    @property
+    def redis_dsn(self) -> RedisDsn:
+        return RedisDsn.build(
+            scheme="redis",
+            host=self.redis_host,
+            port=self.redis_port,
+            path="0",
+        )
+
+
+DB_CONFIG = DbConfig()
+
+__all__ = [
+    "DB_CONFIG",
+]