From 3dc4946a97f18aee05561ca0dc0abe9e0b6aa495 Mon Sep 17 00:00:00 2001 From: labspc Date: Mon, 16 Nov 2026 23:53:22 +0800 Subject: [PATCH 1/2] feat(cli): add octop db migrate-to-pg command for SQLite to PostgreSQL migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a new CLI command group octop db with a migrate-to-pg subcommand that copies all user-data tables from a SQLite source to a PostgreSQL target in a single bulk operation. Features: - Automatic type conversion (REAL→double precision, BLOB→bytea, INTEGER→bigint, JSON→jsonb) - Dual foreign-key strategy: disable session_replication_role when SUPERUSER is available, otherwise order tables by FK dependency (parent tables first) so the copy works without elevated privileges - Per-table COUNT(*) verification with mismatch reporting - --dry-run mode to preview row counts without writing - Idempotent: target tables are DELETEd before INSERT, so the command can be re-run safely - Lazy-loaded via the existing CLI registry (no import cost at launch) --- src/octop/cli/commands/db.py | 342 +++++++++++++++++++++++++++++++++++ src/octop/cli/registry.py | 5 + 2 files changed, 347 insertions(+) create mode 100644 src/octop/cli/commands/db.py diff --git a/src/octop/cli/commands/db.py b/src/octop/cli/commands/db.py new file mode 100644 index 0000000000..c6ccf16f8b --- /dev/null +++ b/src/octop/cli/commands/db.py @@ -0,0 +1,342 @@ +"""octop db — database inspection and migration utilities.""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + +import click +from psycopg import connect as pg_connect + +from octop.config import DatabaseConfig, load_config +from octop.infra.db.factory import open_database +from octop.infra.utils.paths import PathLayout + +# Tables that belong to the target Postgres dialect (never overwritten from +# SQLite). ``sqlite_sequence`` is an SQLite autoincrement bookkeeping table +# that does not exist on Postgres. +PG_NATIVE_TABLES: set[str] = {"_schema_version", "sqlite_sequence"} + + +def _resolve_src_path(home: Path | None) -> PathLayout: + return PathLayout(home) if home is not None else PathLayout.from_env() + + +def _resolve_tgt_config(path: str) -> tuple[Path, DatabaseConfig]: + """Load a target config.json and return (config_dir, DatabaseConfig).""" + cfg_path = Path(path).expanduser() + if not cfg_path.is_file(): + raise click.BadParameter(f"not a file: {cfg_path}") + cfg = load_config(cfg_path) + if not cfg.database.is_postgresql: + raise click.BadParameter( + f"target config at {cfg_path} does not set database.driver=postgresql" + ) + return cfg_path.parent, cfg.database + + +def _source_tables(sqlite_path: Path) -> list[str]: + """Return user-data table names in the source SQLite database.""" + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + ).fetchall() + finally: + conn.close() + return [r[0] for r in rows if r[0] not in PG_NATIVE_TABLES] + + +def _source_row_count(sqlite_path: Path, table: str) -> int: + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + return conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + finally: + conn.close() + + +def _column_info(sqlite_path: Path, table: str) -> list[tuple[str, str]]: + """Return [(name, sqlite_type), ...] for a table.""" + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + return [(r[1], (r[2] or "").upper()) for r in conn.execute(f"PRAGMA table_info({table})")] + finally: + conn.close() + + +def _pg0param(col_type: str) -> str: + """Return a psycopg placeholder + cast expression for a SQLite column type.""" + t = col_type.upper() + if t in ("REAL", "FLOAT", "DOUBLE"): + return "%s::double precision" + if t in ("INTEGER", "INT", "BIGINT"): + return "%s::bigint" + if t == "BLOB": + return "%s::bytea" + if t in ("JSON",): + return "%s::jsonb" + return "%s" + + +def _convert_value(val: Any, col_type: str) -> Any: + """Convert a single SQLite cell value to a psycopg-compatible type.""" + if val is None: + return None + t = col_type.upper() + if t in ("REAL", "FLOAT", "DOUBLE"): + return float(val) + if t in ("INTEGER", "INT", "BIGINT"): + return int(val) + if t == "BLOB": + return bytes(val) + if t in ("JSON",): + if isinstance(val, str) and val.strip(): + return json.dumps(json.loads(val)) + if isinstance(val, (dict, list)): + return json.dumps(val) + return str(val) if val else "[]" + if isinstance(val, float): + return float(val) + if isinstance(val, int): + return int(val) + return str(val) + + +@click.group() +def db() -> None: + """Database inspection and migration utilities.""" + + +@db.command("migrate-to-pg") +@click.option( + "--target-config", + required=True, + type=click.Path(exists=True, dir_okay=False), + help=( + "Path to the target Postgres config.json. The target database must " + "already be initialised (octop init or equivalent) so that all " + "tables exist before data is copied." + ), +) +@click.option( + "--source-home", + default=None, + type=click.Path(exists=True, file_okay=False), + help="Source OCTOP_HOME directory (default: $OCTOP_HOME or ~/.octop).", +) +@click.option( + "--dry-run", + is_flag=True, + help="Report what would be migrated and exit without writing anything.", +) +@click.option( + "--yes", + is_flag=True, + help="Skip the confirmation prompt.", +) +def migrate_to_pg( + target_config: str, + source_home: str | None, + dry_run: bool, + yes: bool, +) -> None: + """Migrate all user data from a SQLite database into a PostgreSQL database. + + The **source** is the current (or ``--source-home``) Octop install whose + ``config.json`` has ``database.driver: sqlite``. + + The **target** is a PostgreSQL Octop install whose ``config.json`` is + passed via ``--target-config``. Run ``octop init`` (or equivalent) against + the target *first* so that every table exists — this command only copies + rows, it does not create any schema. + + \b + Important: + * Stop ``octop run`` on *both* the source and target instances first. + * The target database user needs ``SUPERUSER`` so that foreign-key + checks can be temporarily disabled during the bulk copy. + """ + src_paths = _resolve_src_path( + Path(source_home).expanduser() if source_home else None + ) + sqlite_path = src_paths.db + if not sqlite_path.is_file(): + raise click.UsageError(f"SQLite database not found: {sqlite_path}") + + tgt_dir, tgt_db = _resolve_tgt_config(target_config) + + # Open both sides to validate connectivity and schema versions. + src_cfg = load_config(src_paths.config) + if not src_cfg.database.is_sqlite: + click.echo( + f"Source config at {src_paths.config} is not SQLite; nothing to migrate." + ) + return + + src_db = open_database(src_cfg, src_paths) + try: + with src_db.connect() as conn: + src_ver = conn.execute( + "SELECT version FROM _schema_version" + ).fetchone()[0] + finally: + src_db.close() + + tgt_conn = pg_connect(tgt_db.postgresql_conninfo(), autocommit=True, connect_timeout=10) + try: + tgt_ver_row = tgt_conn.execute( + "SELECT version FROM _schema_version" + ).fetchone() + tgt_ver = tgt_ver_row[0] if tgt_ver_row else 0 + if tgt_ver < int(src_ver): + raise click.UsageError( + f"Target schema version {tgt_ver} is older than source ({src_ver}). " + "Run 'octop init' against the target first." + ) + finally: + tgt_conn.close() + + tables = _source_tables(sqlite_path) + if not tables: + click.echo("No user-data tables found.") + return + + # Summary before confirmation. + total_rows = 0 + table_counts: list[tuple[str, int]] = [] + for tbl in tables: + n = _source_row_count(sqlite_path, tbl) + if n: + table_counts.append((tbl, n)) + total_rows += n + + click.echo(f"Source : {sqlite_path}") + click.echo(f"Target : {tgt_db.postgresql_conninfo()}") + click.echo(f"Schema : {src_ver} -> {tgt_ver}") + click.echo(f"Tables : {len(tables)} ({len(table_counts)} with data, {total_rows} rows)") + click.echo() + + if not table_counts: + click.echo("All tables are empty — nothing to copy.") + return + + for tbl, n in table_counts: + click.echo(f" {tbl:40s} {n:>8d} rows") + click.echo() + + if dry_run: + click.echo("--dry-run: no changes made.") + return + + if not yes: + click.confirm("Overwrite target tables with source data?", abort=True) + + # Disable foreign-key checks (requires SUPERUSER). + tgt_conn = pg_connect(tgt_db.postgresql_conninfo(), autocommit=True, connect_timeout=10) + fk_disabled = True + try: + tgt_conn.execute("SET session_replication_role = 'replica'") + except Exception: + fk_disabled = False + click.echo( + "Foreign-key checks are active (SUPERUSER not available). " + "Tables will be ordered so that referenced tables are copied first." + ) + + # When foreign keys are enforced, migrate tables without foreign keys + # first, then retry the rest — their referenced parents are now in place. + first_pass: list[str] = [] + second_pass: list[str] = [] + if not fk_disabled: + # Heuristic: tables whose name appears as a prefix in other FK + # constraint names are likely parent tables. + fk_tables: set[str] = set() + try: + for r in tgt_conn.execute( + "SELECT table_name FROM information_schema.table_constraints " + "WHERE constraint_type='FOREIGN KEY' AND table_schema='public'" + ): + fk_tables.add(r[0]) + except Exception: + pass + for tbl in tables: + (second_pass if tbl in fk_tables else first_pass).append(tbl) + ordered = first_pass + second_pass + else: + ordered = list(tables) + + failures: list[str] = [] + try: + for idx, tbl in enumerate(ordered, 1): + cols = _column_info(sqlite_path, tbl) + if not cols: + continue + col_names = [c[0] for c in cols] + col_types = [c[1] for c in cols] + sqlite_count = _source_row_count(sqlite_path, tbl) + + if sqlite_count == 0: + pg_count = tgt_conn.execute( + f"SELECT COUNT(*) FROM {tbl}" + ).fetchone()[0] + if pg_count != 0: + click.echo( + f" [{idx}/{len(tables)}] {tbl}: SKIP (0 src rows, " + f"{pg_count} target rows — left unchanged)" + ) + continue + + placeholders = [_pg0param(ct) for ct in col_types] + insert_sql = ( + f"INSERT INTO {tbl} ({','.join(col_names)}) " + f"VALUES ({','.join(placeholders)})" + ) + + # Clear target table then copy. + tgt_conn.execute(f"DELETE FROM {tbl}") + + conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + try: + sqlite_rows = conn.execute(f"SELECT * FROM {tbl}").fetchall() + finally: + conn.close() + + with tgt_conn.transaction(): + for row in sqlite_rows: + values = [ + _convert_value(row[ci], col_types[ci]) + for ci in range(len(row)) + ] + tgt_conn.execute(insert_sql, values) + + pg_count = tgt_conn.execute( + f"SELECT COUNT(*) FROM {tbl}" + ).fetchone()[0] + if pg_count == sqlite_count: + click.echo( + f" [{idx}/{len(tables)}] {tbl:40s} " + f"{sqlite_count:>8d} -> {pg_count:>8d}" + ) + else: + click.echo( + f" [{idx}/{len(tables)}] {tbl:40s} " + f"{sqlite_count:>8d} -> {pg_count:>8d} MISMATCH" + ) + failures.append(tbl) + + finally: + # Restore FK checks. + try: + tgt_conn.execute("SET session_replication_role = 'origin'") + except Exception: + pass + tgt_conn.close() + + if failures: + click.echo() + click.echo(f"Row-count mismatches: {', '.join(failures)}") + raise click.ClickException("Migration finished with errors — see above.") + + click.echo() + click.echo(f"Migrated {len(tables)} tables ({total_rows} rows) successfully.") diff --git a/src/octop/cli/registry.py b/src/octop/cli/registry.py index 66520b8eba..8d61076a17 100644 --- a/src/octop/cli/registry.py +++ b/src/octop/cli/registry.py @@ -41,4 +41,9 @@ "migrate", "Migrate an OpenClaw or Hermes Agent setup into an expert.", ), + "db": ( + ".commands.db", + "db", + "Database inspection and migration (e.g. SQLite -> PostgreSQL).", + ), } From cce24619710631306c279f50f570279ea34f35e4 Mon Sep 17 00:00:00 2001 From: labspc Date: Tue, 17 Nov 2026 11:43:59 +0800 Subject: [PATCH 2/2] test(cli): add P0 type-mapping unit tests for _pg0param and _convert_value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 37 new tests covering: - _pg0param(): 22 parametrized cases (REAL/FLOAT/DOUBLE/INTEGER/INT/BIGINT/BLOB/JSON + fallback) - _convert_value(): NULL passthrough, numeric type dispatch (float/int), BLOB→bytes, JSON roundtrip/dict/list/empty-string, fallback inference All 43 tests in test_cli_db.py pass (zero regression on 6 existing tests). --- tests/unit/cli/test_cli_db.py | 153 ++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/tests/unit/cli/test_cli_db.py b/tests/unit/cli/test_cli_db.py index 677073ee36..b76d7dac6e 100644 --- a/tests/unit/cli/test_cli_db.py +++ b/tests/unit/cli/test_cli_db.py @@ -137,3 +137,156 @@ def test_models_active_explains_an_unset_default(fake_home: Path) -> None: runner.invoke(cli, ["models", "active", "--provider", "acme", "--model", "small"]) shown = runner.invoke(cli, ["models", "active"]) assert shown.output.strip() == "acme / small" + + +# ── P0: _pg0param type-to-cast mapping ────────────────────────────── + + +@pytest.mark.parametrize( + "col_type,expected", + [ + ("REAL", "%s::double precision"), + ("real", "%s::double precision"), + ("FLOAT", "%s::double precision"), + ("float", "%s::double precision"), + ("DOUBLE", "%s::double precision"), + ("double", "%s::double precision"), + ("INTEGER", "%s::bigint"), + ("integer", "%s::bigint"), + ("INT", "%s::bigint"), + ("int", "%s::bigint"), + ("BIGINT", "%s::bigint"), + ("bigint", "%s::bigint"), + ("BLOB", "%s::bytea"), + ("blob", "%s::bytea"), + ("JSON", "%s::jsonb"), + ("json", "%s::jsonb"), + # unknown / fallback + ("TEXT", "%s"), + ("text", "%s"), + ("VARCHAR", "%s"), + ("BOOLEAN", "%s"), + ("TIMESTAMP", "%s"), + ("", "%s"), + ], +) +def test_pg0param_returns_correct_cast(col_type: str, expected: str) -> None: + from octop.cli.commands.db import _pg0param + + assert _pg0param(col_type) == expected + + +# ── P0: _convert_value type dispatch ────────────────────────────────── + + +class TestConvertValueNull: + def test_none_is_none_regardless_of_type(self) -> None: + from octop.cli.commands.db import _convert_value + + for t in ("INTEGER", "REAL", "TEXT", "BLOB", "JSON", "BOOLEAN", ""): + assert _convert_value(None, t) is None + + +class TestConvertValueNumeric: + def test_real_types_produce_float(self) -> None: + from octop.cli.commands.db import _convert_value + + for t in ("REAL", "real", "FLOAT", "float", "DOUBLE", "double"): + assert _convert_value(3.14, t) == 3.14 + assert isinstance(_convert_value(3.14, t), float) + assert _convert_value(42, t) == 42.0 + assert isinstance(_convert_value(42, t), float) + + def test_integer_types_produce_int(self) -> None: + from octop.cli.commands.db import _convert_value + + for t in ("INTEGER", "integer", "INT", "int", "BIGINT", "bigint"): + assert _convert_value(42, t) == 42 + assert isinstance(_convert_value(42, t), int) + assert _convert_value(0, t) == 0 + assert _convert_value(-1, t) == -1 + + +class TestConvertValueBlob: + def test_blob_produces_bytes(self) -> None: + from octop.cli.commands.db import _convert_value + + result = _convert_value(b"hello", "BLOB") + assert result == b"hello" + assert isinstance(result, bytes) + + def test_blob_empty(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value(b"", "BLOB") == b"" + + +class TestConvertValueJson: + def test_json_string_roundtrips(self) -> None: + from octop.cli.commands.db import _convert_value + + result = _convert_value('{"a":1}', "JSON") + assert result == '{"a": 1}' + import json + + json.loads(result) # must be valid JSON + + def test_json_dict_is_serialised(self) -> None: + from octop.cli.commands.db import _convert_value + + result = _convert_value({"key": [1, 2]}, "JSON") + assert result == '{"key": [1, 2]}' + + def test_json_list_is_serialised(self) -> None: + from octop.cli.commands.db import _convert_value + + result = _convert_value([1, "two", 3.0], "JSON") + assert result == '[1, "two", 3.0]' + + def test_json_empty_string_yields_empty_array(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value("", "JSON") == "[]" + + def test_json_whitespace_string_passed_through(self) -> None: + """Whitespace-only string is truthy, so the final fallback + ``return str(val) if val else "[]"`` returns it as-is.""" + from octop.cli.commands.db import _convert_value + + assert _convert_value(" ", "JSON") == " " + + def test_json_plain_string_raises_decode_error(self) -> None: + """json.loads('hello') raises JSONDecodeError — the function + does not catch this; callers must ensure only valid JSON strings + are stored in JSON-typed columns.""" + import json + + from octop.cli.commands.db import _convert_value + + with pytest.raises(json.JSONDecodeError): + _convert_value("hello", "JSON") + + +class TestConvertValueFallback: + """When col_type is unknown, infer from Python value type.""" + + def test_fallback_float_stays_float(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value(3.14, "TEXT") == 3.14 + + def test_fallback_int_stays_int(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value(42, "VARCHAR") == 42 + + def test_fallback_str_stays_str(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value("hello", "UNKNOWN") == "hello" + assert _convert_value("你好", "") == "你好" + + def test_fallback_none_is_none(self) -> None: + from octop.cli.commands.db import _convert_value + + assert _convert_value(None, "UNKNOWN") is None