From dd23ca8723926d262745efa9b13072dbcaaacadf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:11:00 +0000 Subject: [PATCH 1/3] fix(db): make fresh-database schema bootstrap work end-to-end The retired 'emails' table (replaced by 'email_records' during the email model reconciliation) was still referenced by fresh-DB setup, breaking both 'alembic upgrade head' and bootstrap_db against a clean database: - schema_backfill_sql() created a dead 'ix_emails_owner_date ON emails' index (used by migration 0001 and bootstrap_db) -> UndefinedTableError. - migration 0011_email_read_state did 'ALTER TABLE emails ADD COLUMN is_read' unconditionally; guard it on the table existing (matching the has_table/ has_column pattern used by later revisions) since email_records already carries is_read from the model metadata. - give email_records.is_read a server_default so create_all/bootstrap_db match the migration intent and raw inserts that omit is_read (postgres smoke seeds) don't hit a NOT NULL violation. Co-authored-by: Seongho Bae --- .../alembic/versions/0011_email_read_state.py | 20 +++++++++++++++++++ backend/db/models.py | 8 +++++++- backend/scripts/bootstrap_db.py | 4 ---- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/backend/alembic/versions/0011_email_read_state.py b/backend/alembic/versions/0011_email_read_state.py index 716590cd1..2e9395493 100644 --- a/backend/alembic/versions/0011_email_read_state.py +++ b/backend/alembic/versions/0011_email_read_state.py @@ -14,6 +14,19 @@ def upgrade() -> None: + # ``emails`` was the pre-reconciliation email table. ``email_records`` is now + # the single source of truth and already declares ``is_read`` via the model + # metadata created in 0001, so this legacy column add only applies to older + # databases that still carry the retired ``emails`` table. Guard on the table + # existing (matching the has_table/has_column pattern used by later + # revisions) so ``alembic upgrade head`` succeeds on fresh databases. + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("emails"): + return + existing_columns = {column["name"] for column in inspector.get_columns("emails")} + if "is_read" in existing_columns: + return op.add_column( "emails", sa.Column( @@ -26,4 +39,11 @@ def upgrade() -> None: def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("emails"): + return + existing_columns = {column["name"] for column in inspector.get_columns("emails")} + if "is_read" not in existing_columns: + return op.drop_column("emails", "is_read") diff --git a/backend/db/models.py b/backend/db/models.py index 98e17eef2..a86ac0b15 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -18,6 +18,7 @@ Text, UniqueConstraint, func, + text, ) from sqlalchemy.orm import declarative_base, Mapped, mapped_column, relationship from sqlalchemy.types import TypeDecorator @@ -803,7 +804,12 @@ def owner_filters(cls, user_id: str, organization_id: str | None): date: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), index=True) body: Mapped[str] = mapped_column(Text) # IMAP \Seen read state; defaults read so historical/file imports don't nag. - is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # A DB-level server_default keeps create_all/bootstrap_db consistent with the + # 0011_email_read_state migration intent so raw inserts that omit is_read + # (e.g. postgres smoke seeds) don't hit a NOT NULL violation. + is_read: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default=text("true") + ) # Defer large pgvector payloads on default entity loads. embedding = mapped_column(Vector(1536), deferred=True) attachments: Mapped[list["Attachment"]] = relationship( diff --git a/backend/scripts/bootstrap_db.py b/backend/scripts/bootstrap_db.py index 1047103e8..3e579a053 100644 --- a/backend/scripts/bootstrap_db.py +++ b/backend/scripts/bootstrap_db.py @@ -186,10 +186,6 @@ def _get_create_indexes_statements() -> list[Executable]: "CREATE INDEX IF NOT EXISTS ix_email_records_owner_date " "ON email_records (user_id, organization_id, date)" ), - text( - "CREATE INDEX IF NOT EXISTS ix_emails_owner_date " - "ON emails (user_id, organization_id, date)" - ), text( "CREATE INDEX IF NOT EXISTS ix_sender_relationships_owner_source " "ON sender_relationships " From d8ef79b6ee0f3482d62becdaae282801a1433a62 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:11:00 +0000 Subject: [PATCH 2/3] fix(email-import): avoid NUL byte in Postgres advisory-lock key The owner import quota advisory lock built its owner key as f'{user_id}\x00{organization_id}' and passed it to hashtext() as a text bind param. PostgreSQL text cannot encode NUL (0x00), so every email import 500'd on real Postgres with CharacterNotInRepertoireError (mocked/SQLite unit tests skip the advisory-lock path, hiding it). Derive a NUL-free sha256 digest instead and update the tests to assert the NUL-free contract. Co-authored-by: Seongho Bae --- backend/services/email_import_service.py | 16 ++++++++++++++-- backend/tests/test_emails_api.py | 15 +++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backend/services/email_import_service.py b/backend/services/email_import_service.py index 1ff9a2bb3..aed37db8d 100644 --- a/backend/services/email_import_service.py +++ b/backend/services/email_import_service.py @@ -243,6 +243,18 @@ def _session_uses_postgresql(session: AsyncSession) -> bool: return getattr(getattr(bind, "dialect", None), "name", None) == "postgresql" +def _owner_import_quota_lock_key(user_id: str, organization_id: str) -> str: + # PostgreSQL text (and therefore hashtext()) cannot encode NUL (0x00), so a + # raw ``user_id\x00organization_id`` separator raises + # CharacterNotInRepertoireError on real Postgres. Derive a NUL-free, + # collision-resistant digest instead (same approach as the content-graph + # source-record uids below). + payload = "\x00".join((user_id, organization_id)) + return hashlib.sha256( + payload.encode("utf-8", errors="surrogatepass") + ).hexdigest() + + async def _acquire_owner_import_quota_lock( session: AsyncSession, *, user_id: str, organization_id: str ) -> bool: @@ -250,7 +262,7 @@ async def _acquire_owner_import_quota_lock( return False lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( @@ -269,7 +281,7 @@ async def _release_owner_import_quota_lock( ) -> None: lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( diff --git a/backend/tests/test_emails_api.py b/backend/tests/test_emails_api.py index 7bffa6ff7..d6862f82f 100644 --- a/backend/tests/test_emails_api.py +++ b/backend/tests/test_emails_api.py @@ -22,6 +22,7 @@ import datetime from unittest.mock import AsyncMock, patch from services.embedding import STORAGE_EMBEDDING_DIMENSION +from services.email_import_service import _owner_import_quota_lock_key from services.email_service import generate_email_fingerprint pytestmark = pytest.mark.usefixtures("dev_auth_dependency_overrides") @@ -1277,14 +1278,18 @@ async def test_import_email_files_serializes_quota_with_postgres_owner_lock( assert "pg_advisory_unlock" in advisory_queries[-1] assert "hashtext(:namespace_key)" in advisory_queries[0] assert ":owner_key" in advisory_queries[0] + # PostgreSQL text/hashtext cannot encode NUL (0x00); the owner key must be a + # NUL-free digest, not a raw ``user\x00org`` separator. + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ] @@ -1340,14 +1345,16 @@ async def test_import_email_files_rejects_when_owner_quota_is_exhausted( advisory_queries = advisory_query_texts(session) assert "pg_advisory_lock" in advisory_queries[0] assert "pg_advisory_unlock" in advisory_queries[-1] + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ] From 40690170283357ea6478fd49b6a7a84cdd5aaeeb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:11:00 +0000 Subject: [PATCH 3/3] chore(env): add Cloud Agent dev environment (backend + frontend + Postgres/pgvector) Repo-managed .cursor/environment.json plus idempotent install/start scripts: - install.sh: system packages (postgresql-16 + pgvector, python venv/build tools), backend venv + pinned requirements, frontend pnpm@11.5.3 deps. - start.sh: bring up the Postgres cluster, generate a per-VM dev .env with random secrets on first boot, ensure the app DB + pgvector extension, and apply alembic migrations. - terminals run the backend (start_backend.py) and frontend (next dev). Co-authored-by: Seongho Bae --- .cursor/environment.json | 20 +++++++++ .cursor/install.sh | 43 ++++++++++++++++++++ .cursor/start.sh | 88 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 .cursor/environment.json create mode 100755 .cursor/install.sh create mode 100755 .cursor/start.sh diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 000000000..5964ed32d --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,20 @@ +{ + "name": "Naruon dev (backend + frontend + Postgres/pgvector)", + "user": "ubuntu", + "install": "bash .cursor/install.sh", + "start": "bash .cursor/start.sh", + "terminals": [ + { + "name": "backend", + "command": "cd backend && . .venv/bin/activate && python scripts/start_backend.py --host 0.0.0.0 --port 8000" + }, + { + "name": "frontend", + "command": "cd frontend && corepack pnpm@11.5.3 dev --port 3000 --hostname 0.0.0.0" + } + ], + "ports": [ + { "name": "backend", "port": 8000 }, + { "name": "frontend", "port": 3000 } + ] +} diff --git a/.cursor/install.sh b/.cursor/install.sh new file mode 100755 index 000000000..8799ad9bd --- /dev/null +++ b/.cursor/install.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Idempotent repository bootstrap for Naruon Cloud Agent environments. +# +# Prepares durable, source-derived state after checkout: +# * system packages (PostgreSQL 16 + pgvector, Python venv/build tooling) +# * the backend virtualenv + pinned Python requirements +# * the frontend pnpm dependency tree +# +# Per-boot service startup (Postgres, schema migrations, dev secrets) lives in +# .cursor/start.sh so it re-runs on every VM boot, including builds. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +echo "==> [install] system packages (postgresql-16, pgvector, python venv, build tools)" +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get install -y -qq \ + postgresql-16 \ + postgresql-16-pgvector \ + postgresql-client-16 \ + python3.12-venv \ + python3.12-dev \ + build-essential + +echo "==> [install] backend virtualenv + requirements" +cd "$REPO_ROOT/backend" +if [ ! -x ".venv/bin/python" ]; then + python3 -m venv .venv +fi +# shellcheck disable=SC1091 +. .venv/bin/activate +python -m pip install --upgrade pip +pip install -r requirements.txt + +echo "==> [install] frontend dependencies (pnpm@11.5.3)" +cd "$REPO_ROOT/frontend" +corepack enable +corepack prepare pnpm@11.5.3 --activate +corepack pnpm@11.5.3 install --frozen-lockfile + +echo "==> [install] done" diff --git a/.cursor/start.sh b/.cursor/start.sh new file mode 100755 index 000000000..61d9aac3a --- /dev/null +++ b/.cursor/start.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Per-boot runtime reconciliation for Naruon Cloud Agent environments. +# +# Runs on every VM start (idempotent): brings up the PostgreSQL cluster, +# materializes a local dev .env with generated secrets on first boot, ensures +# the app database + pgvector extension exist, and applies Alembic migrations. +# +# Dependency installation lives in .cursor/install.sh; this script only +# reconciles per-boot state and then returns so the backend/frontend terminals +# can start. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +ENV_FILE="$HOME/.env" +PY="$REPO_ROOT/backend/.venv/bin/python" + +echo "==> [start] ensuring PostgreSQL 16 cluster is online" +if ! pg_lsclusters -h 2>/dev/null | awk '{print $4}' | grep -q online; then + sudo pg_ctlcluster 16 main start || true +fi +# Wait for the server to accept connections before touching it. +for _ in $(seq 1 30); do + if sudo -u postgres pg_isready -q; then break; fi + sleep 1 +done + +echo "==> [start] generating local dev .env on first boot (secrets are per-VM)" +if [ ! -f "$ENV_FILE" ]; then + "$PY" - "$ENV_FILE" <<'PYGEN' +import os, secrets, sys +from pathlib import Path +from cryptography.fernet import Fernet + +env_path = Path(sys.argv[1]) +db_password = secrets.token_urlsafe(24) +hmac_secret = secrets.token_urlsafe(48) +enc_key = Fernet.generate_key().decode() + +env_path.write_text( + "# Naruon local dev environment (generated per-VM; not committed).\n" + f"DATABASE_URL=postgresql+asyncpg://postgres:{db_password}@127.0.0.1:5432/ai_email\n" + f"AUTH_SESSION_HMAC_SECRET={hmac_secret}\n" + f"ENCRYPTION_KEY={enc_key}\n" + "DEBUG=false\n" + "RUNTIME_ENVIRONMENT=development\n" + "ENABLE_PROMETHEUS_METRICS=false\n" + "ALLOWED_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000," + "http://localhost:8000,http://127.0.0.1:8000\n" + "SMTP_MODE=simulated\n" + "OPENAI_API_KEY=\n" + "OPENAI_EMBEDDING_MODEL=text-embedding-3-small\n" + "OPENAI_MODEL=gpt-4o\n", + encoding="utf-8", +) +os.chmod(env_path, 0o600) +print(f"wrote {env_path}") +PYGEN +fi + +echo "==> [start] reconciling database role, database, and pgvector extension" +DB_PASSWORD="$( + "$PY" - "$ENV_FILE" <<'PYPW' +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + +for line in Path(sys.argv[1]).read_text(encoding="utf-8").splitlines(): + if line.startswith("DATABASE_URL="): + print(unquote(urlsplit(line.split("=", 1)[1]).password or "")) + break +PYPW +)" +# Keep the local postgres role password in sync with the generated DATABASE_URL. +sudo -u postgres psql -v ON_ERROR_STOP=1 \ + -c "ALTER USER postgres WITH PASSWORD '${DB_PASSWORD}';" >/dev/null +if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='ai_email'" | grep -q 1; then + sudo -u postgres createdb ai_email +fi +sudo -u postgres psql -d ai_email -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null + +echo "==> [start] applying database migrations (alembic upgrade head)" +cd "$REPO_ROOT/backend" +"$PY" scripts/migrate_db.py + +echo "==> [start] done"