diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 000000000..5964ed32d --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,20 @@ +{ + "name": "Naruon dev (backend + frontend + Postgres/pgvector)", + "user": "ubuntu", + "install": "bash .cursor/install.sh", + "start": "bash .cursor/start.sh", + "terminals": [ + { + "name": "backend", + "command": "cd backend && . .venv/bin/activate && python scripts/start_backend.py --host 0.0.0.0 --port 8000" + }, + { + "name": "frontend", + "command": "cd frontend && corepack pnpm@11.5.3 dev --port 3000 --hostname 0.0.0.0" + } + ], + "ports": [ + { "name": "backend", "port": 8000 }, + { "name": "frontend", "port": 3000 } + ] +} diff --git a/.cursor/install.sh b/.cursor/install.sh new file mode 100755 index 000000000..8799ad9bd --- /dev/null +++ b/.cursor/install.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Idempotent repository bootstrap for Naruon Cloud Agent environments. +# +# Prepares durable, source-derived state after checkout: +# * system packages (PostgreSQL 16 + pgvector, Python venv/build tooling) +# * the backend virtualenv + pinned Python requirements +# * the frontend pnpm dependency tree +# +# Per-boot service startup (Postgres, schema migrations, dev secrets) lives in +# .cursor/start.sh so it re-runs on every VM boot, including builds. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +echo "==> [install] system packages (postgresql-16, pgvector, python venv, build tools)" +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get install -y -qq \ + postgresql-16 \ + postgresql-16-pgvector \ + postgresql-client-16 \ + python3.12-venv \ + python3.12-dev \ + build-essential + +echo "==> [install] backend virtualenv + requirements" +cd "$REPO_ROOT/backend" +if [ ! -x ".venv/bin/python" ]; then + python3 -m venv .venv +fi +# shellcheck disable=SC1091 +. .venv/bin/activate +python -m pip install --upgrade pip +pip install -r requirements.txt + +echo "==> [install] frontend dependencies (pnpm@11.5.3)" +cd "$REPO_ROOT/frontend" +corepack enable +corepack prepare pnpm@11.5.3 --activate +corepack pnpm@11.5.3 install --frozen-lockfile + +echo "==> [install] done" diff --git a/.cursor/start.sh b/.cursor/start.sh new file mode 100755 index 000000000..61d9aac3a --- /dev/null +++ b/.cursor/start.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Per-boot runtime reconciliation for Naruon Cloud Agent environments. +# +# Runs on every VM start (idempotent): brings up the PostgreSQL cluster, +# materializes a local dev .env with generated secrets on first boot, ensures +# the app database + pgvector extension exist, and applies Alembic migrations. +# +# Dependency installation lives in .cursor/install.sh; this script only +# reconciles per-boot state and then returns so the backend/frontend terminals +# can start. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +ENV_FILE="$HOME/.env" +PY="$REPO_ROOT/backend/.venv/bin/python" + +echo "==> [start] ensuring PostgreSQL 16 cluster is online" +if ! pg_lsclusters -h 2>/dev/null | awk '{print $4}' | grep -q online; then + sudo pg_ctlcluster 16 main start || true +fi +# Wait for the server to accept connections before touching it. +for _ in $(seq 1 30); do + if sudo -u postgres pg_isready -q; then break; fi + sleep 1 +done + +echo "==> [start] generating local dev .env on first boot (secrets are per-VM)" +if [ ! -f "$ENV_FILE" ]; then + "$PY" - "$ENV_FILE" <<'PYGEN' +import os, secrets, sys +from pathlib import Path +from cryptography.fernet import Fernet + +env_path = Path(sys.argv[1]) +db_password = secrets.token_urlsafe(24) +hmac_secret = secrets.token_urlsafe(48) +enc_key = Fernet.generate_key().decode() + +env_path.write_text( + "# Naruon local dev environment (generated per-VM; not committed).\n" + f"DATABASE_URL=postgresql+asyncpg://postgres:{db_password}@127.0.0.1:5432/ai_email\n" + f"AUTH_SESSION_HMAC_SECRET={hmac_secret}\n" + f"ENCRYPTION_KEY={enc_key}\n" + "DEBUG=false\n" + "RUNTIME_ENVIRONMENT=development\n" + "ENABLE_PROMETHEUS_METRICS=false\n" + "ALLOWED_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000," + "http://localhost:8000,http://127.0.0.1:8000\n" + "SMTP_MODE=simulated\n" + "OPENAI_API_KEY=\n" + "OPENAI_EMBEDDING_MODEL=text-embedding-3-small\n" + "OPENAI_MODEL=gpt-4o\n", + encoding="utf-8", +) +os.chmod(env_path, 0o600) +print(f"wrote {env_path}") +PYGEN +fi + +echo "==> [start] reconciling database role, database, and pgvector extension" +DB_PASSWORD="$( + "$PY" - "$ENV_FILE" <<'PYPW' +import sys +from pathlib import Path +from urllib.parse import unquote, urlsplit + +for line in Path(sys.argv[1]).read_text(encoding="utf-8").splitlines(): + if line.startswith("DATABASE_URL="): + print(unquote(urlsplit(line.split("=", 1)[1]).password or "")) + break +PYPW +)" +# Keep the local postgres role password in sync with the generated DATABASE_URL. +sudo -u postgres psql -v ON_ERROR_STOP=1 \ + -c "ALTER USER postgres WITH PASSWORD '${DB_PASSWORD}';" >/dev/null +if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='ai_email'" | grep -q 1; then + sudo -u postgres createdb ai_email +fi +sudo -u postgres psql -d ai_email -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null + +echo "==> [start] applying database migrations (alembic upgrade head)" +cd "$REPO_ROOT/backend" +"$PY" scripts/migrate_db.py + +echo "==> [start] done" diff --git a/backend/alembic/versions/0011_email_read_state.py b/backend/alembic/versions/0011_email_read_state.py index 716590cd1..2e9395493 100644 --- a/backend/alembic/versions/0011_email_read_state.py +++ b/backend/alembic/versions/0011_email_read_state.py @@ -14,6 +14,19 @@ def upgrade() -> None: + # ``emails`` was the pre-reconciliation email table. ``email_records`` is now + # the single source of truth and already declares ``is_read`` via the model + # metadata created in 0001, so this legacy column add only applies to older + # databases that still carry the retired ``emails`` table. Guard on the table + # existing (matching the has_table/has_column pattern used by later + # revisions) so ``alembic upgrade head`` succeeds on fresh databases. + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("emails"): + return + existing_columns = {column["name"] for column in inspector.get_columns("emails")} + if "is_read" in existing_columns: + return op.add_column( "emails", sa.Column( @@ -26,4 +39,11 @@ def upgrade() -> None: def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("emails"): + return + existing_columns = {column["name"] for column in inspector.get_columns("emails")} + if "is_read" not in existing_columns: + return op.drop_column("emails", "is_read") diff --git a/backend/db/models.py b/backend/db/models.py index 98e17eef2..a86ac0b15 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -18,6 +18,7 @@ Text, UniqueConstraint, func, + text, ) from sqlalchemy.orm import declarative_base, Mapped, mapped_column, relationship from sqlalchemy.types import TypeDecorator @@ -803,7 +804,12 @@ def owner_filters(cls, user_id: str, organization_id: str | None): date: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), index=True) body: Mapped[str] = mapped_column(Text) # IMAP \Seen read state; defaults read so historical/file imports don't nag. - is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # A DB-level server_default keeps create_all/bootstrap_db consistent with the + # 0011_email_read_state migration intent so raw inserts that omit is_read + # (e.g. postgres smoke seeds) don't hit a NOT NULL violation. + is_read: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default=text("true") + ) # Defer large pgvector payloads on default entity loads. embedding = mapped_column(Vector(1536), deferred=True) attachments: Mapped[list["Attachment"]] = relationship( diff --git a/backend/scripts/bootstrap_db.py b/backend/scripts/bootstrap_db.py index 1047103e8..3e579a053 100644 --- a/backend/scripts/bootstrap_db.py +++ b/backend/scripts/bootstrap_db.py @@ -186,10 +186,6 @@ def _get_create_indexes_statements() -> list[Executable]: "CREATE INDEX IF NOT EXISTS ix_email_records_owner_date " "ON email_records (user_id, organization_id, date)" ), - text( - "CREATE INDEX IF NOT EXISTS ix_emails_owner_date " - "ON emails (user_id, organization_id, date)" - ), text( "CREATE INDEX IF NOT EXISTS ix_sender_relationships_owner_source " "ON sender_relationships " diff --git a/backend/services/email_import_service.py b/backend/services/email_import_service.py index 1ff9a2bb3..aed37db8d 100644 --- a/backend/services/email_import_service.py +++ b/backend/services/email_import_service.py @@ -243,6 +243,18 @@ def _session_uses_postgresql(session: AsyncSession) -> bool: return getattr(getattr(bind, "dialect", None), "name", None) == "postgresql" +def _owner_import_quota_lock_key(user_id: str, organization_id: str) -> str: + # PostgreSQL text (and therefore hashtext()) cannot encode NUL (0x00), so a + # raw ``user_id\x00organization_id`` separator raises + # CharacterNotInRepertoireError on real Postgres. Derive a NUL-free, + # collision-resistant digest instead (same approach as the content-graph + # source-record uids below). + payload = "\x00".join((user_id, organization_id)) + return hashlib.sha256( + payload.encode("utf-8", errors="surrogatepass") + ).hexdigest() + + async def _acquire_owner_import_quota_lock( session: AsyncSession, *, user_id: str, organization_id: str ) -> bool: @@ -250,7 +262,7 @@ async def _acquire_owner_import_quota_lock( return False lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( @@ -269,7 +281,7 @@ async def _release_owner_import_quota_lock( ) -> None: lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( diff --git a/backend/tests/test_emails_api.py b/backend/tests/test_emails_api.py index 7bffa6ff7..d6862f82f 100644 --- a/backend/tests/test_emails_api.py +++ b/backend/tests/test_emails_api.py @@ -22,6 +22,7 @@ import datetime from unittest.mock import AsyncMock, patch from services.embedding import STORAGE_EMBEDDING_DIMENSION +from services.email_import_service import _owner_import_quota_lock_key from services.email_service import generate_email_fingerprint pytestmark = pytest.mark.usefixtures("dev_auth_dependency_overrides") @@ -1277,14 +1278,18 @@ async def test_import_email_files_serializes_quota_with_postgres_owner_lock( assert "pg_advisory_unlock" in advisory_queries[-1] assert "hashtext(:namespace_key)" in advisory_queries[0] assert ":owner_key" in advisory_queries[0] + # PostgreSQL text/hashtext cannot encode NUL (0x00); the owner key must be a + # NUL-free digest, not a raw ``user\x00org`` separator. + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ] @@ -1340,14 +1345,16 @@ async def test_import_email_files_rejects_when_owner_quota_is_exhausted( advisory_queries = advisory_query_texts(session) assert "pg_advisory_lock" in advisory_queries[0] assert "pg_advisory_unlock" in advisory_queries[-1] + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ]