From a827aee1838bd2f9835ea2611d13eb0e77a36718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=94=E6=AF=94?= Date: Wed, 14 Oct 2026 12:38:30 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E5=9C=A8=E9=82=8F=E8=BC=AF=E6=90=AC?= =?UTF-8?q?=E7=A7=BB=E8=AE=80=E5=87=BA=E5=A4=A7=E5=9E=8B=E6=AC=84=E4=BD=8D?= =?UTF-8?q?=E5=89=8D=E9=99=90=E5=88=B6=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原本先 GetValue 再檢查大小，超大 BLOB 已配置完整陣列才拒絕。改成 SQL 長度與 CASE 限額投影，先檢查單欄及累計預算，才讀取有界內容；PostgreSQL 分段讀取，SQLite 避免 expression 欄位的 GetBytes metadata 原生錯誤。隔離 SQLite 證明 32 MiB 欄位不配置完整 managed 陣列且空目標不變，1 MiB 邊界可正常搬移。 --- .../Services/LogicalMigrationService.cs | 45 ++++++++++++++++--- .../LogicalMigrationTests.cs | 14 ++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/mySQLPunk.Core/Services/LogicalMigrationService.cs b/mySQLPunk.Core/Services/LogicalMigrationService.cs index 696722d..fd6b7bc 100644 --- a/mySQLPunk.Core/Services/LogicalMigrationService.cs +++ b/mySQLPunk.Core/Services/LogicalMigrationService.cs @@ -14,6 +14,7 @@ namespace MySqlPunk.Core.Services; public static class LogicalMigrationService { public const int MaximumRows = 100_000; + internal const int MaximumFieldBytes = 1024 * 1024; private const long MaximumBytes = 64 * 1024 * 1024; private static readonly Regex Identifier = new("\\A[A-Za-z_][A-Za-z0-9_]{0,62}\\z", RegexOptions.CultureInvariant); private sealed record Snapshot(string Version, string Identity, List Tables, @@ -238,20 +239,28 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D if (readData && blockers.Count == 0) { await using var command = connection.CreateCommand(); command.Transaction = transaction; - command.CommandText = "SELECT " + string.Join(",", columns.Select(column => Quote(column.Name))) + " FROM " + Qualified(provider, definition.Name); - await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + command.CommandText = "SELECT " + string.Join(",", columns.Select(column => + { + var name = Quote(column.Name); + if (column.Kind == "integer") return name; + var length = (provider == DatabaseProviderKind.Sqlite ? "length(" : "octet_length(") + name + ")"; + return length + ",CASE WHEN " + length + " <= " + MaximumFieldBytes + " THEN " + name + " ELSE NULL END"; + })) + " FROM " + Qualified(provider, definition.Name); + await using var reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess, token).ConfigureAwait(false); while (await reader.ReadAsync(token).ConfigureAwait(false)) { if (++totalRows > MaximumRows) throw new InvalidOperationException("來源超過 100,000 列，請使用專用搬移器。"); var row = new object?[columns.Count]; + var ordinal = 0; for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) { - var value = reader.IsDBNull(columnIndex) ? null : reader.GetValue(columnIndex); - if (value != null && (columns[columnIndex].Kind == "integer" ? value is not long : value is not byte[])) - throw new InvalidOperationException("實際儲存型別與宣告不符。"); - var bytes = value is byte[] binary ? binary.Length : 8; + var binary = columns[columnIndex].Kind == "binary"; + var bytes = reader.IsDBNull(ordinal) ? 0 : binary ? reader.GetInt64(ordinal) : 8; totalBytes += bytes; - if (bytes > 1024 * 1024 || totalBytes > MaximumBytes) throw new InvalidOperationException("超過單欄 1 MiB／總量 64 MiB 預覽上限。"); + if (bytes > MaximumFieldBytes || totalBytes > MaximumBytes) throw new InvalidOperationException("超過單欄 1 MiB／總量 64 MiB 預覽上限。"); + if (binary) ordinal++; + object? value = reader.IsDBNull(ordinal) ? null : binary ? ReadBoundedBytes(reader, ordinal, bytes, token) : reader.GetInt64(ordinal); + ordinal++; row[columnIndex] = value; } rows.Add(row); @@ -263,6 +272,28 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D return new(version, identity, tables, data, blockers); } + internal static byte[] ReadBoundedBytes(DbDataReader reader, int ordinal, long length, CancellationToken token) + { + if (length < 0 || length > MaximumFieldBytes) throw new InvalidOperationException("超過單欄 1 MiB 預覽上限。"); + token.ThrowIfCancellationRequested(); + if (reader is SqliteDataReader) + { + var bounded = reader.GetFieldValue(ordinal); + if (bounded.LongLength != length) throw new InvalidOperationException("欄位長度在讀取期間改變。"); + return bounded; + } + var result = new byte[(int)length]; + var offset = 0; + while (offset < result.Length) + { + token.ThrowIfCancellationRequested(); + var read = (int)reader.GetBytes(ordinal, offset, result, offset, Math.Min(8192, result.Length - offset)); + if (read == 0) throw new InvalidOperationException("欄位長度在讀取期間改變。"); + offset += read; + } + return result; + } + internal const string PostgreSqlUnsupportedCatalogSql = """ SELECT (SELECT count(*) FROM pg_namespace WHERE nspname NOT IN ('public','information_schema','pg_catalog') AND nspname NOT LIKE 'pg_toast%' AND nspname NOT LIKE 'pg_temp_%') + diff --git a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs index fbba9e5..f60d0b5 100644 --- a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs +++ b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs @@ -63,6 +63,20 @@ public static async Task RunAsync() catch (SqliteException) { Check(!File.Exists(missing.Database), "No automatic creation of missing target"); } var unsupported = source.Clone(); unsupported.Provider = DatabaseProviderKind.MySql; Check(!(await LogicalMigrationService.PreviewAsync(unsupported, reverse, 3)).CanStart, "Unsupported engine refused without opening a network connection"); + var large = Profile(Path.Combine(root, "large.db")); + using (var owner = DatabaseProviderFactory.Create(large)) + { + await owner.ExecuteAsync(large.Database, "CREATE TABLE large (payload BLOB) STRICT; INSERT INTO large VALUES (zeroblob(33554432));"); + var empty = Profile(Path.Combine(root, "large-target.db")); CreateEmpty(empty.Database); + var allocatedBefore = GC.GetTotalAllocatedBytes(true); + try { await LogicalMigrationService.PreviewAsync(large, empty, 3); throw new Exception("Oversize BLOB must be denied"); } + catch (InvalidOperationException error) { Check(error.Message.Contains("1 MiB"), "Length refusal before materialization"); } + Check(GC.GetTotalAllocatedBytes(true) - allocatedBefore < 8 * 1024 * 1024, "32 MiB BLOB is not materialized in managed memory"); + Check(await ObjectCountAsync(empty) == 0, "Oversize read leaves target empty"); + await owner.ExecuteAsync(large.Database, "UPDATE large SET payload=zeroblob(1048576);"); + var boundary = await LogicalMigrationService.PreviewAsync(large, empty, 3); + Check(boundary.CanStart && (await LogicalMigrationService.RunAsync(large, empty, 3, boundary, empty.Database)).Outcome == LogicalMigrationOutcome.Verified, "1 MiB inclusive boundary transfers in bounded chunks"); + } foreach (var definition in new[] { "CREATE TABLE bad (id INTEGER PRIMARY KEY, payload BLOB) STRICT;", From f855298a40922a89bd20908df7bc4ae102b76628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=94=E6=AF=94?= Date: Wed, 14 Oct 2026 12:50:26 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=E4=BF=9D=E7=95=99=E9=82=8F?= =?UTF-8?q?=E8=BC=AF=E6=90=AC=E7=A7=BB=E7=9A=84=E6=96=87=E5=AD=97=E4=B8=BB?= =?UTF-8?q?=E9=8D=B5=E8=88=87=E6=99=AE=E9=80=9A=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第一階段只有整數與二進位且拒絕所有鍵索引，無法搬移帶文字的常見業務表。新增 SQLite BINARY 與 PostgreSQL C 的有效 UTF-8 文字映射、明確非自增主鍵與升冪普通索引，正式 catalog 檢查定序、opclass、NULL 排序及特殊索引；不支援的語意仍在寫入前拒絕。 主鍵與索引加入確認 fingerprint、DDL 預覽及提交前結構驗證，索引建立後取消與索引遭修改都整批回滾。隔離 SQLite 與正式 Avalonia handlers 已通過文字、PK、索引與有界讀取回歸，PG16 fixture 新增雙向業務表與 catalog 測試，由本次 CI 驗證。矩陣分開說明 MySQL/MariaDB 的 implicit DDL commit、SQLServer 的定序與型別轉換障礙，不宣稱全引擎完成，也不發版。 --- CHANGELOG.md | 2 + README.md | 2 +- docs/FEATURE_NOTES.md | 2 +- docs/LOGICAL_MIGRATION_MATRIX.md | 42 +++++--- mySQLPunk.Core/Models/LogicalMigration.cs | 7 +- .../Services/LogicalMigrationKeys.cs | 97 +++++++++++++++++++ .../Services/LogicalMigrationService.cs | 69 +++++++++---- .../LogicalMigrationTests.cs | 77 ++++++++++++++- mySQLPunk.Desktop/LogicalMigrationWindow.cs | 4 +- .../DesktopProfileOperationHarness/Program.cs | 12 ++- 10 files changed, 274 insertions(+), 40 deletions(-) create mode 100644 mySQLPunk.Core/Services/LogicalMigrationKeys.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index c09e2f4..acd2d60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### 🚀 新增功能 +- **邏輯搬移的文字、主鍵與索引子集**：SQLite STRICT ↔ PostgreSQL 保留有效 UTF-8 TEXT（BINARY/C）、明確非自增單欄／複合主鍵、普通升冪索引與欄位順序。預覽包含索引 DDL，來源變更失效、取消、缺索引或內容不符仍整批回滾；FK、其他定序、default、identity、特殊索引依舊預檢拒絕，不宣稱所有跨引擎組合完成。 - **可驗證邏輯搬移的第一階段**：SQL Server 專用相容工具加入受限雙向跨版本；Avalonia 新增 SQLite STRICT ↔ PostgreSQL 的整數／二進位搬移，預覽來源／目標版本、schema/data 轉換與阻擋原因，確認後才寫入另建空目標。單一交易、分批、內容校驗與不確定狀態禁止重試，不將一般資料傳輸或原生備份宣稱為無限制跨版本／跨引擎還原。 - **SQL Server 降版相容搬移（Windows，有限範圍預覽版）**：選取實際目標版本，預覽結構與列數並輸入目標名稱確認後，只寫入另建的空資料庫。僅 dbo 一般 heap 與白名單型別；identity、constraints、indexes、triggers、procs、schema 相依與進階功能明確阻擋，定序必須相同。分批寫入單一交易，內容與結構驗證通過才提交，不自動重試；2016–2025 尚僅離線／mock 驗證，不宣稱真引擎矩陣通過。 @@ -53,6 +54,7 @@ - **Linux／macOS SSH Tunnel**：跨平台連線設定可勾選「透過 SSH Tunnel 連線」，填入 SSH 主機、port、使用者、密碼或 OpenSSH／PEM 私鑰（含密語）與必填的主機金鑰 SHA256 指紋；連線時以 SSH.NET 建立只綁定 127.0.0.1 動態 port 的本機轉送，MySQL／MariaDB、PostgreSQL、SQL Server 的所有查詢與資料表操作都改走轉送端點，session 中斷或程式關閉時一併關閉；SSH 連線被跳板機中斷（閒置逾時、重啟、網路中斷）後，下一次資料庫操作會以同樣的指紋規則自動重建轉送，而不是把死掉的端點交給驅動程式。主機金鑰與指紋完全比對、不符即中止並回報伺服器實際指紋；MD5 或格式錯誤的指紋、缺主機／使用者／指紋、非絕對路徑私鑰、私鑰權限過寬與 MySQL／PostgreSQL 的 VerifyFull（127.0.0.1 主機名稱必定不符）一律 fail closed，SQL Server 則改以原始主機名稱驗證憑證。SSH 密碼與私鑰密語不寫入 `connections.json`（設定檔含這兩個欄位會拒絕載入），勾選系統密碼庫時與資料庫密碼一樣以獨立項目存入 Linux Secret Service／macOS Keychain，否則只保留到本次程式關閉；smoke test 會以本機臨時 sshd 實機驗證金鑰登入、加密私鑰、錯誤密語、指紋不符、權限過寬與 provider session 整合，實機矩陣另會經 tunnel 對 MySQL 8／MariaDB 11.4（Required）、PostgreSQL 16（VerifyCA＋CA）與 SQL Server 2022（Mandatory＋伺服器憑證，以原始主機名稱驗證）實際查詢與列出資料庫。 ### 🛠️ 問題修正與優化 +- 邏輯搬移先取得 byte length 與 SQL 限額投影再讀取 BLOB／文字，避免超過 1 MiB 才拒絕時已配置整個大陣列；單欄與累計預算在 materialize 前檢查，超限保持空目標。 - SQLite SQL 與執行計畫改在背景執行，長查詢透過原生 progress handler 中斷，不再凍結 UI；取消後等實際語句停止才釋放操作鎖。取消訊息明示先前已提交語句不會自動回滾，避免盲目重試。 - SQL Server 相容搬移改用既有連線設定工廠，正常 SQL authentication 不再受已開啟連線隱藏密碼影響，SSH tunnel 與 provider owner 一起釋放。 diff --git a/README.md b/README.md index bd97d1a..349daad 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Windows 完整版介面支援繁體中文與英文；資料庫密碼、SSH 密 目前發版版本：`v1.0.0.22`，最新版請看 [GitHub Releases](https://github.com/shadowjohn/mySQLPunk/releases)。 -未發版的可驗證邏輯搬移：Windows SQL Server 專用工具支援 2016–2025 受限雙向跨版本；Avalonia「邏輯搬移」提供 SQLite STRICT ↔ PostgreSQL 的 INTEGER/bigint、BLOB/bytea 子集。只接受另建空目標、明確確認、單一交易與內容校驗；PK/FK、索引、default、identity、字串定序、時間與其他未忠實轉換的功能在預檢阻擋，不是任意引擎／版本的原生 restore。支援方向與實機／mock 證據區分見 [搬移矩陣](docs/LOGICAL_MIGRATION_MATRIX.md)。 +未發版的可驗證邏輯搬移：Windows SQL Server 專用工具支援 2016–2025 受限雙向跨版本；Avalonia「邏輯搬移」提供 SQLite STRICT ↔ PostgreSQL 的 INTEGER/bigint、BLOB/bytea、UTF-8 TEXT（BINARY/C）、非自增單欄／複合主鍵及普通升冪索引子集。只接受另建空目標、明確確認、單一交易與結構／內容校驗；FK、default、identity、其他定序、時間與未忠實轉換的功能在預檢阻擋。MySQL/MariaDB、SQL Server 的跨引擎路徑仍須專用語意與交易轉換，不宣稱整體完成或任意引擎／版本的原生 restore。支援方向與實機／mock 證據區分見 [搬移矩陣](docs/LOGICAL_MIGRATION_MATRIX.md)。 目前的 `v1.0.0.22` GitHub Release 同時提供 Windows x64 完整版 setup、self-contained Linux x64／ARM64 安裝壓縮檔，以及 macOS Intel／Apple Silicon `.app.zip`。Linux 壓縮檔內附使用者層級的 `install.sh`／`uninstall.sh`；macOS 預覽目前採 ad-hoc 簽署、尚未 Apple notarize。每個跨平台資產都有獨立 `.sha256` 可驗證完整性。完整變更請見 `CHANGELOG.md`。 diff --git a/docs/FEATURE_NOTES.md b/docs/FEATURE_NOTES.md index 0516ae9..19c2304 100644 --- a/docs/FEATURE_NOTES.md +++ b/docs/FEATURE_NOTES.md @@ -626,4 +626,4 @@ # 可驗證雙向與跨引擎邏輯搬移 -Windows SQL Server 專用工具加入受限升版／降版／同版本方向；Avalonia 工作台新增 SQLite STRICT 與 PostgreSQL 的 INTEGER/bigint、BLOB/bytea 搬移。兩種工具都只寫入明確確認的新空目標，不把未支援型別、鍵、索引、預設值、trigger/procedure 等靜默丟掉。方向、實際版本、交易／取消／不確定重試及測試矩陣見 [LOGICAL_MIGRATION_MATRIX.md](LOGICAL_MIGRATION_MATRIX.md)。一般資料傳輸與 SQL 文字轉換不等於此處的可驗證邏輯還原。 +Windows SQL Server 專用工具加入受限升版／降版／同版本方向；Avalonia 工作台提供 SQLite STRICT 與 PostgreSQL 的 INTEGER/bigint、BLOB/bytea、有效 UTF-8 TEXT BINARY/C、非自增 PK 與普通升冪索引搬移。兩種工具都只寫入明確確認的新空目標，不把未支援型別、鍵、索引、預設值、trigger/procedure 等靜默丟掉。大型欄位先 SQL 長度／限額投影再讀值，索引及主鍵納入 fingerprint 與正式 catalog 驗證。方向、實際版本、交易／取消／不確定重試及測試矩陣見 [LOGICAL_MIGRATION_MATRIX.md](LOGICAL_MIGRATION_MATRIX.md)。一般資料傳輸與 SQL 文字轉換不等於此處的可驗證邏輯還原。 diff --git a/docs/LOGICAL_MIGRATION_MATRIX.md b/docs/LOGICAL_MIGRATION_MATRIX.md index 3f8ba35..d867376 100644 --- a/docs/LOGICAL_MIGRATION_MATRIX.md +++ b/docs/LOGICAL_MIGRATION_MATRIX.md @@ -7,37 +7,57 @@ | 來源 → 目標 | 程式允許的版本 | 可執行範圍 | 驗證區分 | | --- | --- | --- | --- | | SQL Server → SQL Server | 2016/2017/2019/2022/2025 任意雙向及同版本，共 25 組 | Windows 既有專用工具；dbo heap、既有型別白名單、同定序、無 advanced features | 25 組預檢及交易 mock；不是 25 組真引擎驗證 | -| SQLite → SQLite | 3.37–3.53 | STRICT、INTEGER/BLOB、NULL/NOT NULL | 本機實際 SQLite 3.53.3；歷史 minor 版本未逐一跑 | -| SQLite → PostgreSQL | SQLite 3.37–3.53 → PostgreSQL 14–17 | INTEGER → bigint、BLOB → bytea | 既有隔離 CI PostgreSQL 16 fixture 有雙向內容 round-trip 測試；以該 SHA 的 CI 結果為準 | -| PostgreSQL → SQLite | PostgreSQL 14–17 → SQLite 3.37–3.53 | bigint → INTEGER、bytea → BLOB | 同上；14/15/17 未實機跑 | -| PostgreSQL → PostgreSQL | 14–17 雙向及同版本 | 同一 bigint/bytea 子集與全部預檢 | 邏輯共用路徑；沒有完整跨版本真機矩陣 | +| SQLite → SQLite | 3.37–3.53 | STRICT、INTEGER/BLOB/TEXT、NULL/NOT NULL、非自增 PK、普通索引；詳細子集見下文 | 本機實際 SQLite 3.53.3；歷史 minor 版本未逐一跑 | +| SQLite → PostgreSQL | SQLite 3.37–3.53 → PostgreSQL 14–17 | INTEGER → bigint、BLOB → bytea、TEXT BINARY → text C；非自增 PK、普通索引 | 隔離 CI PostgreSQL 16 fixture 加入文字／鍵／索引雙向 round-trip；只以這次新 head 結果為證據 | +| PostgreSQL → SQLite | PostgreSQL 14–17 → SQLite 3.37–3.53 | 上述型別反向映射；PK → WITHOUT ROWID、普通索引 | 同上；14/15/17 未實機跑 | +| PostgreSQL → PostgreSQL | 14–17 雙向及同版本 | 同一 bigint/bytea/text C、PK／索引子集與全部預檢 | 邏輯共用路徑；沒有完整跨版本真機矩陣 | + +## 第二階段：實用文字與鍵索引子集 + +實作適用明確手動給 ID 的業務表：例如 `customers(id INTEGER NOT NULL, label TEXT COLLATE BINARY NOT NULL, note TEXT, payload BLOB, PRIMARY KEY(id)) STRICT, WITHOUT ROWID`，可帶 `customer_label(label)`、`customer_note(note,id)` 普通索引；複合主鍵如 `lines(customer,code)` 同樣保留順序、唯一性及 NOT NULL。所有索引 DDL 都列在確認預覽，建立後正式 catalog 再讀回驗證，缺索引／改欄位順序也會整批回滾。 + +- SQLite 文字只接受 BINARY（隱含或明確），PostgreSQL 必須是內建 `pg_catalog."C"`；兩端 UTF-8，逐欄嚴格解碼，拒絕 NUL、無效 UTF-8，不偷偷替換字元。保留空字串、NULL、Unicode、大小寫與 byte-order 比較語意；不承諾兩引擎字串函式或 LIKE 行為相同。 +- SQLite PK 表要求完整可辨識 grammar、明確 NOT NULL 與 WITHOUT ROWID，避免 INTEGER PRIMARY KEY 的 rowid alias 自動給 ID 行為被偷偷改成 PostgreSQL 無 sequence 的 bigint。單欄整數／文字／二進位與複合 PK 均可；不保證引擎自動產生的 PK constraint／內部索引名稱或實體配置相同。 +- 普通非 unique btree 索引保留名稱、欄位與順序，只允許升冪、預設內建 opclass、無 expression／partial／INCLUDE／自訂 storage。SQLite 升冪 NULL 在前，目標 PostgreSQL 明確 `ASC NULLS FIRST`；反向 nullable 欄位索引若為 PostgreSQL 預設 NULLS LAST 則拒絕，NOT NULL 索引沒有這項差异。DESC、unique（PK 除外）、自訂定序／opclass、clustered／replica identity 索引都阻擋。 +- SQLite／PostgreSQL schema＋索引＋資料仍為單一交易，不引入逐表 partial success。每次重新預檢／最終驗證涵蓋 PK、索引、內容；已提交或 ack lost 不可盲重試。 +- 有界讀取先在 SQL 投影取 byte length、超限 CASE 回傳 NULL，再检查預算，才 materialize 欄位。BLOB 不會先 GetValue 完整大陣列；SQLite expression 用已限額的 byte[] 讀取（避開 provider GetBytes 的 metadata 路徑），PostgreSQL 分段讀取。這是應用端快照限額，不是對引擎自身 page cache／SQL 執行記憶體的硬上限。 SQLite／PostgreSQL 工具入口在 Avalonia 工作台「邏輯搬移」。來源使用目前連線的記憶體設定；目標選已儲存的連線及另建空資料庫，明確選版本，必要密碼只由使用者輸入，不額外讀秘密庫。獨立 provider owner 保有 SSH tunnel，所有開啟／失敗／結束都有 scope 釋放；不從 redacted ConnectionString 重建。 ## 寫入前安全檢查 -- SQLite 檔案必須已存在；來源唯讀開啟，目標 ReadWrite（不自動建立），使用 private cache、停用 pooling。缺檔、非 UTF-8、非 STRICT、virtual table、生成欄位、default、check、PK/FK、index、trigger、view、內部 sequence 等一律拒絕。 -- SQLite CREATE TABLE 只接受可證明完整的 INTEGER/BLOB STRICT grammar；不是從欄位猜 DDL 後丟掉未辨識 clause。PostgreSQL 查正式 catalog：額外 schema、extension、view、sequence、constraint、index、default、identity、trigger（含 database event trigger）、function/procedure、rule、policy、user type、partition、inheritance、RLS、unlogged、large object、publication/subscription、自訂定序／轉換／運算子／文字搜尋物件等都阻擋。查詢失敗也不放行；不自動搬移帳號、權限或安全設定。 -- 名稱限制 ASCII identifier 63 字元；轉到 SQLite 另阻擋只差大小寫的表／欄位。文字 encoding/collation、時間與浮點／decimal precision、identity/sequence 的忠實轉換尚未實作，因此不是靜默轉成 TEXT 或捨棄鍵／預設值。 +- SQLite 檔案必須已存在；來源唯讀開啟，目標 ReadWrite（不自動建立），使用 private cache、停用 pooling。缺檔、非 UTF-8、非 STRICT、virtual table、生成欄位、default、check、FK、不支援的鍵／索引、trigger、view、內部 sequence 等一律拒絕。 +- SQLite CREATE TABLE／INDEX 只接受可證明完整的安全 grammar；不是從欄位猜 DDL 後丟掉未辨識 clause。PostgreSQL 查正式 catalog，只有上述 PK／普通索引白名單放行：額外 schema、extension、view、sequence、其他 constraint/index、default、identity、trigger（含 database event trigger）、function/procedure、rule、policy、user type、partition、inheritance、RLS、unlogged、large object、publication/subscription、自訂定序／轉換／運算子／文字搜尋物件等都阻擋。查詢失敗也不放行；不自動搬移帳號、權限或安全設定。 +- 名稱限制 ASCII identifier 63 字元；轉到 SQLite 另阻擋只差大小寫的表／欄位／全域索引名稱。其他 encoding/collation、時間與浮點／decimal precision、identity/sequence 仍拒絕，不靜默轉成 TEXT 或捨棄鍵／預設值。 - 預覽顯示實際版本、方向、目標、表、列數、轉換 SQL 與阻擋原因，不顯示資料值。來源內容也納入確認 fingerprint；輸入完整目標名稱才可執行，確認後來源內容（包括相同列數下的變更）／結構／實際引擎識別／目標變動會拒絕。 - SQLite／PostgreSQL 全部 DDL＋資料在單一目標交易。SQLite 用 immediate 交易；PostgreSQL 持有工具專用 advisory lock，來源全部表 SHARE lock 後讀取並在提交前再檢查。這不是在線無鎖搬移；請用隔離副本／維護時段，外部工具不會自動遵守 advisory lock，也不能宣稱能禁止其他管理者在最後檢查之後建立額外物件。 -- 每批至多 100 列／200 參數，最多 100,000 列；每欄至多 1 MiB，每輪快照資料至多 64 MiB。驗證 schema、列數、Int64／binary／NULL 的 SHA-256 多重集合（保留重複列）。不保證隱含 rowid、實體排序或儲存配置跨引擎相同。 +- 每批至多 100 列／200 參數，最多 100,000 列；每欄至多 1 MiB，每輪快照資料至多 64 MiB。驗證 schema、PK／索引、列數、Int64／text／binary／NULL 的 SHA-256 多重集合（保留重複列）。不保證隱含 rowid、實體排序或儲存配置跨引擎相同。 - 取消等實際工作停止，SQLite 用原生 progress handler，PostgreSQL 使用 provider cancellation；工作台 gate 持續到對話框關閉，不能同時 SQL／edit/delete/connect。變更目標／版本／認證即失效原預覽。 ## 重試與恢復 沒有逐表 checkpoint：全部驗證後才提交。失敗且確認回滾的結果是 `RolledBack`，須重新預檢；已驗證提交是 `Verified`，不再寫入。commit 開始後失敗、回覆遺失或回滾失敗是 `Uncertain`，UI 停止再次預檢／寫入，禁止盲目重試，必須由 DBA 核對；不宣稱放棄 Task 就已停止或回滾。已有內容的目標即使是前次結果也不能自動 resume。 -## 尚未實作、執行前拒絕 +## 其他方向：目前阻擋與必要技術工作 -MySQL/MariaDB、SQL Server 與 SQLite/PostgreSQL 之間的通用 verified conversion 尚未接入；MySQL 目標的 implicit-commit DDL 不能沿用此單一交易保證，需另一套可驗證的 staging／恢復設計。Windows Oracle／Snowflake 也未加入此 verified 路徑。MongoDB、Redis/Garnet 等非關聯式 model 沒有通用無損表格映射，不能當同義還原目標。選不支援的 provider，服務在開連線前明確拒絕。 +| 方向 | 現況／執行前拒絕的技術理由 | 下一階段必要能力（不是本輪已通過） | +| --- | --- | --- | +| MySQL/MariaDB → SQLite/PG/SQLServer | 整數 unsigned 上界、zero date、ENUM/SET、charset、PAD SPACE 定序、AUTO_INCREMENT 與 storage engine 一致快照不能用現有 count-only transfer 當忠實轉換 | 來源 catalog／版本／InnoDB 快照 adapter，逐欄區間及 charset/定序證明；不可先沿用一般 type map 再補驗證 | +| SQLite/PG/SQLServer → MySQL/MariaDB，以及 MySQL↔MariaDB／跨版本 | MySQL DDL implicit commit，DDL＋資料單一交易會是假保證；目標即使原本空白，取消仍可能已留下表 | 另建 staging 與 durable uncertain/checkpoint 協議、預檢引擎版本差異、人工核對恢復；不能自動刪 staging 或覆蓋目標假裝 rollback | +| SQLServer → SQLite/PG/MySQL，以及反向 | UTF-16/UTF-8、linguistic/BIN2 定序的比較／尾空白規則、rowversion、decimal/time scale、identity/default/sequence 不能把現有型別名稱替換當成保留語意 | 可重用 SQLServer catalog 的阻擋規則，但需跨 provider canonical schema／資料 codec、來源一致快照與逐欄語意／版本證據；目前只 Windows SQLServer→SQLServer 專用子集可執行 | +| Oracle/Snowflake 與其他引擎 | NUMBER／時間、空字串 NULL、identifier/collation 與交易 DDL 不等价 | 專用 catalog／codec／交易恢復設計；本輪不開連線 | +| MongoDB、Redis/Garnet 與表格引擎 | document/key-value model、TTL／原子操作／巢狀內容沒有通用無損表格映射 | 明確專用模型轉換契約，不當成同義 restore | -有 PK/FK、索引、字串定序、時間、decimal、identity、procedure/trigger 的一般資料庫不能在此階段搬移；替代路徑是引擎官方 schema/data migration 工具與人工轉換／驗證，不是宣稱 bacpac 可無限制降版。新版 SQL Server .bak 仍不能直接還原到舊引擎，compatibility level 不改備份格式；既有原生 restore 預檢繼續保留。 +上述方向服務在開連線前拒絕，沒有聲稱某個 MySQL/MariaDB 或 SQLServer 版本已跨引擎通過。雙向跨版本／換引擎的整體需求仍未全部完成；此批是實際擴充 SQLite/PG 的可驗證第二階段，而不是全引擎完成或發版的宣告。 + +有 FK、其他字串定序、時間、decimal、identity、procedure/trigger 的一般資料庫仍須使用引擎官方 schema/data migration 工具與人工轉換／驗證，不是宣稱 bacpac 可無限制降版。新版 SQL Server .bak 仍不能直接還原到舊引擎，compatibility level 不改備份格式；既有原生 restore 預檢繼續保留。 ## 回歸證據 實際隔離 SQLite：1,005 列、多批、Int64 極值、空／非空 bytes、NULL、重複列、反向內容一致、錯誤確認、缺目標不自動建立、來源同列數內容改變、unsupported features、部分寫入失敗、內容／結構驗證失敗、途中取消，以及真正 commit 後注入 ack lost（保留資料且禁止重試）。不是單靠手動設 mock Blockers。 +本輪新增正式 SQLite：32 MiB BLOB 在 materialize 前拒絕且 managed allocation 小於 8 MiB、1 MiB 邊界搬移、有效 Unicode／空字串／NULL／NUL／無效 UTF-8／文字超限、單欄與複合 PK 真的拒絕重複及缺 ID、索引缺失／順序變动與索引建立後取消整批回滾。PostgreSQL 16 CI fixture 用同一業務表雙向搬移與 C 定序／複合 PK 實際檢查，另對 expression/unique/null ordering/collation 真 catalog 加入與移除後預檢。 + Avalonia/Xvfb：正式入口持有主視窗 gate、預覽、目標變更失效、錯誤確認不寫入、實際確認寫入且禁止重複 resume。既有跨平台 CI fixture 另測 SQLite ↔ PostgreSQL 16，包括正式 catalog 逐項加 feature／移除 feature 後恢復可預檢。 真 SQL Server 跨版本、SQL auth+SSH 真機、PostgreSQL 14/15/17、SQLite 歷史版本及非關聯式轉換皆沒有完整矩陣證據；不得把 mock／同版本一般 provider round-trip 當全部資料庫版本已通過。 diff --git a/mySQLPunk.Core/Models/LogicalMigration.cs b/mySQLPunk.Core/Models/LogicalMigration.cs index a196ef1..07fa9b4 100644 --- a/mySQLPunk.Core/Models/LogicalMigration.cs +++ b/mySQLPunk.Core/Models/LogicalMigration.cs @@ -1,7 +1,12 @@ namespace MySqlPunk.Core.Models; public sealed record LogicalMigrationColumn(string Name, string Kind, bool Nullable); -public sealed record LogicalMigrationTable(string Name, IReadOnlyList Columns, long Rows, string ContentHash); +public sealed record LogicalMigrationIndex(string Name, IReadOnlyList Columns); +public sealed record LogicalMigrationTable(string Name, IReadOnlyList Columns, long Rows, string ContentHash) +{ + public IReadOnlyList PrimaryKey { get; init; } = Array.Empty(); + public IReadOnlyList Indexes { get; init; } = Array.Empty(); +} public sealed record LogicalMigrationPreview( string SourceVersion, string TargetVersion, string TargetDatabase, IReadOnlyList Tables, IReadOnlyList Statements, diff --git a/mySQLPunk.Core/Services/LogicalMigrationKeys.cs b/mySQLPunk.Core/Services/LogicalMigrationKeys.cs new file mode 100644 index 0000000..3ed10ea --- /dev/null +++ b/mySQLPunk.Core/Services/LogicalMigrationKeys.cs @@ -0,0 +1,97 @@ +using System.Data.Common; +using System.Text.RegularExpressions; +using MySqlPunk.Core.Models; + +namespace MySqlPunk.Core.Services; + +public static partial class LogicalMigrationService +{ + private sealed record Keys(IReadOnlyList PrimaryKey, IReadOnlyList Indexes); + + private static async Task ReadKeysAsync(DbConnection connection, DbTransaction? transaction, + DatabaseProviderKind provider, string table, List columns, List blockers, CancellationToken token) + { + var primary = new List(); + var indexes = new List(); + if (provider == DatabaseProviderKind.Sqlite) + { + var ordered = new SortedDictionary(); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; command.CommandText = "PRAGMA table_xinfo(" + Quote(table) + ");"; + await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + if (reader.GetInt64(5) > 0) ordered.Add(reader.GetInt64(5), reader.GetString(1)); + } + primary.AddRange(ordered.Values); + var definitions = new List<(string Name, bool Unique, string Origin, bool Partial)>(); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; command.CommandText = "PRAGMA index_list(" + Quote(table) + ");"; + await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + definitions.Add((reader.GetString(1), reader.GetInt64(2) != 0, reader.GetString(3), reader.GetInt64(4) != 0)); + } + foreach (var definition in definitions) + { + if (definition.Origin != "pk" && (!Identifier.IsMatch(definition.Name) || definition.Unique || definition.Partial || definition.Origin != "c")) + blockers.Add("不支援 unique／partial／自訂索引：" + definition.Name); + var fields = new List(); + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; command.CommandText = "PRAGMA index_xinfo(" + Quote(definition.Name) + ");"; + await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + { + if (reader.GetInt64(5) == 0) continue; + if (reader.GetInt64(1) < 0 || reader.GetInt64(3) != 0 || reader.GetString(4) != "BINARY") + blockers.Add("索引只允許普通欄位、升冪、BINARY 定序：" + definition.Name); + else fields.Add(reader.GetString(2)); + } + } + if (definition.Origin == "pk") + { + if (!fields.SequenceEqual(primary)) blockers.Add("主鍵索引與欄位順序不符：" + table); + continue; + } + await using (var command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = "SELECT sql FROM sqlite_schema WHERE type='index' AND name=@name;"; + var parameter = command.CreateParameter(); parameter.ParameterName = "@name"; parameter.Value = definition.Name; command.Parameters.Add(parameter); + var sql = Convert.ToString(await command.ExecuteScalarAsync(token).ConfigureAwait(false)) ?? ""; + static string Pattern(string name) => "(?:" + Regex.Escape(name) + "|\"" + Regex.Escape(name) + "\")"; + var grammar = "\\ACREATE\\s+INDEX\\s+" + Pattern(definition.Name) + "\\s+ON\\s+" + Pattern(table) + + "\\s*\\(\\s*" + string.Join("\\s*,\\s*", fields.Select(name => Pattern(name) + "(?:\\s+ASC)?")) + "\\s*\\)\\s*;?\\s*\\z"; + if (!Regex.IsMatch(sql, grammar, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + blockers.Add("索引定義含不支援語意：" + definition.Name); + } + indexes.Add(new(definition.Name, fields.ToArray())); + } + } + else + { + await using var command = connection.CreateCommand(); command.Transaction = transaction; + command.CommandText = "SELECT ic.relname, i.indisprimary, a.attname FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_class ic ON ic.oid=i.indexrelid CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY k(attnum,position) LEFT JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum=k.attnum WHERE n.nspname='public' AND c.relname=@table ORDER BY ic.relname COLLATE \"C\", k.position;"; + var parameter = command.CreateParameter(); parameter.ParameterName = "@table"; parameter.Value = table; command.Parameters.Add(parameter); + await using var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false); + var indexFields = new SortedDictionary>(StringComparer.Ordinal); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + { + if (reader.IsDBNull(2)) { blockers.Add("不支援 expression 索引：" + reader.GetString(0)); continue; } + if (reader.GetBoolean(1)) primary.Add(reader.GetString(2)); + else + { + var name = reader.GetString(0); + if (!Identifier.IsMatch(name)) blockers.Add("索引名稱超出安全子集：" + name); + if (!indexFields.TryGetValue(name, out var fields)) indexFields.Add(name, fields = new()); + fields.Add(reader.GetString(2)); + } + } + indexes.AddRange(indexFields.Select(entry => new LogicalMigrationIndex(entry.Key, entry.Value.ToArray()))); + } + if (primary.Any(name => columns.All(column => column.Name != name || column.Nullable))) + blockers.Add("主鍵必須明確 NOT NULL：" + table); + return new(primary.ToArray(), indexes.OrderBy(index => index.Name, StringComparer.Ordinal).ToArray()); + } +} diff --git a/mySQLPunk.Core/Services/LogicalMigrationService.cs b/mySQLPunk.Core/Services/LogicalMigrationService.cs index fd6b7bc..d43b319 100644 --- a/mySQLPunk.Core/Services/LogicalMigrationService.cs +++ b/mySQLPunk.Core/Services/LogicalMigrationService.cs @@ -11,7 +11,7 @@ namespace MySqlPunk.Core.Services; -public static class LogicalMigrationService +public static partial class LogicalMigrationService { public const int MaximumRows = 100_000; internal const int MaximumFieldBytes = 1024 * 1024; @@ -44,7 +44,7 @@ private static async Task PreviewCoreAsync(ConnectionPr int selectedTargetMajor, CancellationToken token) { if (!Supported(source) || !Supported(target)) - return Refused(target, "此階段只實作 SQLite STRICT 與 PostgreSQL 的整數／二進位邏輯搬移。其他引擎請使用專用搬移器；不可當成原生 restore。"); + return Refused(target, "可驗證跨引擎路徑為 SQLite STRICT ↔ PostgreSQL；MySQL 目標 DDL 隱含提交，SQL Server 定序／型別與 identity 需另行轉換預檢，不能沿用此交易保證或當成原生 restore。請見方向矩陣。"); using var sourceOwner = (AdoDatabaseSession)DatabaseProviderFactory.Create(source); using var targetOwner = (AdoDatabaseSession)DatabaseProviderFactory.Create(target); await using var origin = await sourceOwner.OpenMigrationConnectionAsync(source.Database, token, readOnly: true).ConfigureAwait(false); @@ -72,11 +72,16 @@ private static LogicalMigrationPreview BuildPreview(ConnectionProfile source, Co if (target.Provider == DatabaseProviderKind.Sqlite && (origin.Tables.GroupBy(table => table.Name, StringComparer.OrdinalIgnoreCase).Any(group => group.Count() > 1) || origin.Tables.Any(table => table.Columns.GroupBy(column => column.Name, StringComparer.OrdinalIgnoreCase).Any(group => group.Count() > 1)))) blockers.Add("來源名稱只差大小寫；SQLite 無法忠實保存這些不同的表／欄位名稱。"); + if (target.Provider == DatabaseProviderKind.Sqlite && origin.Tables.Select(table => table.Name) + .Concat(origin.Tables.SelectMany(table => table.Indexes.Select(index => index.Name))) + .GroupBy(name => name, StringComparer.OrdinalIgnoreCase).Any(group => group.Count() > 1)) + blockers.Add("表／索引名稱在 SQLite 全域名稱空間衝突。"); if (Major(destination.Version) != selectedTargetMajor) blockers.Add("明確選取的目標版本與實際引擎不符。"); if (source.Provider == target.Provider && source.Database == target.Database && (source.Provider == DatabaseProviderKind.Sqlite || source.Host == target.Host && source.Port == target.Port)) blockers.Add("來源與目標不得為同一資料庫。"); - var statements = origin.Tables.Select(table => CreateTableSql(target.Provider, table)).ToArray(); + var statements = origin.Tables.Select(table => CreateTableSql(target.Provider, table)) + .Concat(origin.Tables.SelectMany(table => table.Indexes.Select(index => CreateIndexSql(target.Provider, table, index)))).ToArray(); var fingerprint = Hash(JsonSerializer.Serialize(new { Source = new { source.Provider, source.Host, source.Port, source.Database }, @@ -136,7 +141,7 @@ private static async Task RunCoreAsync(ConnectionProfile var name = "@value" + command.Parameters.Count; var parameter = command.CreateParameter(); parameter.ParameterName = name; - parameter.DbType = table.Columns[columnIndex].Kind == "integer" ? DbType.Int64 : DbType.Binary; + parameter.DbType = table.Columns[columnIndex].Kind == "integer" ? DbType.Int64 : table.Columns[columnIndex].Kind == "text" ? DbType.String : DbType.Binary; parameter.Value = row[columnIndex] ?? DBNull.Value; command.Parameters.Add(parameter); parameters.Add(name); @@ -150,6 +155,12 @@ private static async Task RunCoreAsync(ConnectionProfile observer?.Invoke("batch", destination, targetTransaction); } } + foreach (var statement in current.Statements.Skip(sourceSnapshot.Tables.Count)) + { + token.ThrowIfCancellationRequested(); + await ExecuteAsync(destination, targetTransaction, statement, token).ConfigureAwait(false); + observer?.Invoke("index", destination, targetTransaction); + } stage = "逐欄型別、NULL、內容與結構驗證"; observer?.Invoke("verify", destination, targetTransaction); var written = await ReadSnapshotAsync(destination, targetTransaction, target.Provider, true, token).ConfigureAwait(false); @@ -216,6 +227,7 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D while (await reader.ReadAsync(token).ConfigureAwait(false)) { var name = reader.GetString(0); + if (provider == DatabaseProviderKind.Sqlite && reader.GetString(1) == "index") continue; if (reader.GetString(1) != "table" || name.StartsWith("sqlite_", StringComparison.Ordinal)) blockers.Add("不支援物件：" + name + "（" + reader.GetString(1) + "）"); else definitions.Add((name, reader.GetString(2))); @@ -233,8 +245,9 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D if (!Identifier.IsMatch(definition.Name)) { blockers.Add("表名不在 ASCII 63 字元安全子集：" + definition.Name); continue; } var columns = await ReadColumnsAsync(connection, transaction, provider, definition.Name, blockers, token).ConfigureAwait(false); if (columns.Count == 0 || columns.Count > 100) { blockers.Add("欄位數必須為 1–100：" + definition.Name); continue; } - if (provider == DatabaseProviderKind.Sqlite && !SafeSqliteDefinition(definition.Name, columns, definition.Definition)) - blockers.Add("僅接受無鍵／索引／default／collation／生成欄位的 INTEGER、BLOB STRICT 表：" + definition.Name); + var keys = await ReadKeysAsync(connection, transaction, provider, definition.Name, columns, blockers, token).ConfigureAwait(false); + if (provider == DatabaseProviderKind.Sqlite && !SafeSqliteDefinition(definition.Name, columns, definition.Definition, keys.PrimaryKey)) + blockers.Add("表定義超出 INTEGER/BLOB/TEXT BINARY STRICT、明確 NOT NULL 與 WITHOUT ROWID 主鍵子集：" + definition.Name); var rows = new List(); if (readData && blockers.Count == 0) { @@ -243,8 +256,9 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D { var name = Quote(column.Name); if (column.Kind == "integer") return name; - var length = (provider == DatabaseProviderKind.Sqlite ? "length(" : "octet_length(") + name + ")"; - return length + ",CASE WHEN " + length + " <= " + MaximumFieldBytes + " THEN " + name + " ELSE NULL END"; + var bytes = column.Kind == "text" ? provider == DatabaseProviderKind.Sqlite ? "CAST(" + name + " AS BLOB)" : "convert_to(" + name + ",'UTF8')" : name; + var length = (provider == DatabaseProviderKind.Sqlite ? "length(" : "octet_length(") + bytes + ")" + (provider == DatabaseProviderKind.PostgreSql ? "::bigint" : ""); + return length + ",CASE WHEN " + length + " <= " + MaximumFieldBytes + " THEN " + bytes + " ELSE NULL END"; })) + " FROM " + Qualified(provider, definition.Name); await using var reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess, token).ConfigureAwait(false); while (await reader.ReadAsync(token).ConfigureAwait(false)) @@ -254,19 +268,27 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D var ordinal = 0; for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) { - var binary = columns[columnIndex].Kind == "binary"; + var binary = columns[columnIndex].Kind != "integer"; var bytes = reader.IsDBNull(ordinal) ? 0 : binary ? reader.GetInt64(ordinal) : 8; totalBytes += bytes; if (bytes > MaximumFieldBytes || totalBytes > MaximumBytes) throw new InvalidOperationException("超過單欄 1 MiB／總量 64 MiB 預覽上限。"); if (binary) ordinal++; object? value = reader.IsDBNull(ordinal) ? null : binary ? ReadBoundedBytes(reader, ordinal, bytes, token) : reader.GetInt64(ordinal); + if (value is byte[] encoded && columns[columnIndex].Kind == "text") + { + string text; + try { text = new UTF8Encoding(false, true).GetString(encoded); } + catch (DecoderFallbackException) { throw new InvalidOperationException("文字不是有效 UTF-8，不能忠實轉換。"); } + if (text.Contains('\0')) throw new InvalidOperationException("文字含 NUL，PostgreSQL 無法忠實儲存。"); + value = text; + } ordinal++; row[columnIndex] = value; } rows.Add(row); } } - tables.Add(new(definition.Name, columns.AsReadOnly(), rows.Count, ContentHash(rows))); + tables.Add(new(definition.Name, columns.AsReadOnly(), rows.Count, ContentHash(rows)) { PrimaryKey = keys.PrimaryKey, Indexes = keys.Indexes }); data.Add(definition.Name, rows); } return new(version, identity, tables, data, blockers); @@ -313,12 +335,14 @@ internal static byte[] ReadBoundedBytes(DbDataReader reader, int ordinal, long l (SELECT count(*) FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public') + (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='public') + (SELECT count(*) FROM pg_type t JOIN pg_namespace n ON n.oid=t.typnamespace WHERE n.nspname='public' AND t.typtype NOT IN ('c') AND NOT (t.typcategory='A' AND EXISTS (SELECT 1 FROM pg_type base JOIN pg_class c ON c.oid=base.typrelid WHERE base.oid=t.typelem AND c.relkind='r'))) + - (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relkind<>'r' OR c.relpersistence<>'p' OR c.relispartition OR c.relrowsecurity OR c.relforcerowsecurity OR c.reloptions IS NOT NULL OR c.reltablespace<>0)) + - (SELECT count(*) FROM pg_constraint c JOIN pg_namespace n ON n.oid=c.connamespace WHERE n.nspname='public') + + (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relkind NOT IN ('r','i') OR c.relpersistence<>'p' OR c.relispartition OR c.relrowsecurity OR c.relforcerowsecurity OR c.reloptions IS NOT NULL OR c.reltablespace<>0)) + + (SELECT count(*) FROM pg_constraint c JOIN pg_namespace n ON n.oid=c.connamespace WHERE n.nspname='public' AND (c.contype<>'p' OR c.condeferrable OR c.condeferred OR NOT c.convalidated)) + + (SELECT count(*) FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_class ic ON ic.oid=i.indexrelid JOIN pg_am am ON am.oid=ic.relam WHERE n.nspname='public' AND (am.amname<>'btree' OR (i.indisunique AND NOT i.indisprimary) OR NOT i.indisvalid OR NOT i.indisready OR NOT i.indislive OR i.indisclustered OR i.indisreplident OR i.indnkeyatts<>i.indnatts OR i.indexprs IS NOT NULL OR i.indpred IS NOT NULL)) + + (SELECT count(*) FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY k(attnum,position) LEFT JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum=k.attnum LEFT JOIN pg_opclass op ON op.oid=i.indclass[k.position::int-1] LEFT JOIN pg_namespace ons ON ons.oid=op.opcnamespace WHERE n.nspname='public' AND (a.attnum IS NULL OR ons.nspname<>'pg_catalog' OR NOT op.opcdefault OR op.opcname NOT IN ('int8_ops','text_ops','bytea_ops') OR (i.indoption[k.position::int-1] & 1)<>0 OR (NOT i.indisprimary AND NOT a.attnotnull AND (i.indoption[k.position::int-1] & 2)=0) OR i.indcollation[k.position::int-1]<>a.attcollation)) + (SELECT count(*) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public') + (SELECT count(*) FROM pg_attrdef a JOIN pg_class c ON c.oid=a.adrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public') + (SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid=i.inhrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public') + - (SELECT count(*) FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND a.attnum>0 AND (a.attisdropped OR a.attidentity<>'' OR a.attgenerated<>'' OR a.attcollation<>0 OR a.atttypid NOT IN (20,17))); + (SELECT count(*) FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind='r' AND a.attnum>0 AND (a.attisdropped OR a.attidentity<>'' OR a.attgenerated<>'' OR a.atttypid NOT IN (20,17,25) OR (a.atttypid=25 AND a.attcollation<>'pg_catalog."C"'::regcollation) OR (a.atttypid<>25 AND a.attcollation<>0))); """; private static async Task> ReadColumnsAsync(DbConnection connection, DbTransaction? transaction, @@ -334,7 +358,7 @@ private static async Task> ReadColumnsAsync(DbConne { var name = reader.GetString(provider == DatabaseProviderKind.Sqlite ? 1 : 0); var type = reader.GetString(provider == DatabaseProviderKind.Sqlite ? 2 : 1).ToLowerInvariant(); - var kind = type is "integer" or "bigint" ? "integer" : type is "blob" or "bytea" ? "binary" : "unsupported"; + var kind = type is "integer" or "bigint" ? "integer" : type is "blob" or "bytea" ? "binary" : type == "text" ? "text" : "unsupported"; if (!Identifier.IsMatch(name) || kind == "unsupported") blockers.Add("不支援欄位名稱／型別：" + table + "." + name + " " + type); var notNull = provider == DatabaseProviderKind.Sqlite ? reader.GetInt64(3) != 0 : reader.GetBoolean(2); columns.Add(new(name, kind, !notNull)); @@ -342,26 +366,35 @@ private static async Task> ReadColumnsAsync(DbConne return columns; } - private static bool SafeSqliteDefinition(string name, List columns, string sql) + private static bool SafeSqliteDefinition(string name, List columns, string sql, IReadOnlyList primaryKey) { static string NamePattern(string identifier) => "(?:" + Regex.Escape(identifier) + "|\"" + Regex.Escape(identifier) + "\")"; var definitions = columns.Select(column => NamePattern(column.Name) + "\\s+" + - (column.Kind == "integer" ? "INTEGER" : "BLOB") + (column.Nullable ? "" : "\\s+NOT\\s+NULL")); + (column.Kind == "integer" ? "INTEGER" : column.Kind == "text" ? "TEXT(?:\\s+COLLATE\\s+BINARY)?" : "BLOB") + (column.Nullable ? "" : "\\s+NOT\\s+NULL")); + var key = primaryKey.Count == 0 ? "" : "\\s*,\\s*PRIMARY\\s+KEY\\s*\\(\\s*" + string.Join("\\s*,\\s*", primaryKey.Select(NamePattern)) + "\\s*\\)"; return Regex.IsMatch(sql, "\\ACREATE\\s+TABLE\\s+" + NamePattern(name) + "\\s*\\(\\s*" + - string.Join("\\s*,\\s*", definitions) + "\\s*\\)\\s+STRICT\\s*;?\\s*\\z", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + string.Join("\\s*,\\s*", definitions) + key + "\\s*\\)\\s+" + + (primaryKey.Count == 0 ? "STRICT" : "(?:STRICT\\s*,\\s*WITHOUT\\s+ROWID|WITHOUT\\s+ROWID\\s*,\\s*STRICT)") + "\\s*;?\\s*\\z", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); } private static string CreateTableSql(DatabaseProviderKind provider, LogicalMigrationTable table) => "CREATE TABLE " + Qualified(provider, table.Name) + " (" + string.Join(",", table.Columns.Select(column => Quote(column.Name) + " " + (column.Kind == "integer" ? provider == DatabaseProviderKind.Sqlite ? "INTEGER" : "bigint" + : column.Kind == "text" ? provider == DatabaseProviderKind.Sqlite ? "TEXT COLLATE BINARY" : "text COLLATE \"C\"" : provider == DatabaseProviderKind.Sqlite ? "BLOB" : "bytea") + (column.Nullable ? "" : " NOT NULL"))) + - ")" + (provider == DatabaseProviderKind.Sqlite ? " STRICT;" : ";"); + (table.PrimaryKey.Count == 0 ? "" : ",PRIMARY KEY (" + string.Join(",", table.PrimaryKey.Select(Quote)) + ")") + + ")" + (provider == DatabaseProviderKind.Sqlite ? " STRICT" + (table.PrimaryKey.Count == 0 ? ";" : ", WITHOUT ROWID;") : ";"); + + private static string CreateIndexSql(DatabaseProviderKind provider, LogicalMigrationTable table, LogicalMigrationIndex index) => + "CREATE INDEX " + Quote(index.Name) + " ON " + Qualified(provider, table.Name) + " (" + + string.Join(",", index.Columns.Select(name => Quote(name) + (provider == DatabaseProviderKind.PostgreSql ? " ASC NULLS FIRST" : ""))) + ");"; private static int Major(string version) => int.TryParse(version.Split('.')[0], out var major) ? major : -1; private static string Quote(string name) => "\"" + name.Replace("\"", "\"\"") + "\""; private static string Qualified(DatabaseProviderKind provider, string table) => (provider == DatabaseProviderKind.PostgreSql ? "\"public\"." : "") + Quote(table); private static string Hash(string value) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(value))); private static string ContentHash(List rows) => Hash(string.Join("\n", rows.Select(row => Hash(string.Join("|", row.Select(value => value == null ? "null" : value is byte[] binary ? "binary:" + Convert.ToBase64String(binary) + : value is string text ? "text:" + Convert.ToBase64String(Encoding.UTF8.GetBytes(text)) : "integer:" + ((long)value).ToString(CultureInfo.InvariantCulture))))).OrderBy(value => value, StringComparer.Ordinal))); private static async Task ScalarAsync(DbConnection connection, DbTransaction? transaction, string sql, CancellationToken token) { diff --git a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs index f60d0b5..693c770 100644 --- a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs +++ b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs @@ -86,7 +86,11 @@ public static async Task RunAsync() "CREATE TABLE bad (id TEXT COLLATE NOCASE, payload BLOB) STRICT;", "CREATE TABLE bad (id INTEGER, payload BLOB);", "CREATE TABLE bad (id INTEGER REFERENCES sample(number), payload BLOB) STRICT;", - "CREATE TABLE bad (id INTEGER, payload BLOB) STRICT; CREATE INDEX bad_index ON bad(id);", + "CREATE TABLE bad (id INTEGER, payload BLOB) STRICT; CREATE INDEX bad_index ON bad(id DESC);", + "CREATE TABLE bad (id INTEGER NOT NULL, PRIMARY KEY(id)) STRICT;", + "CREATE TABLE bad (id TEXT NOT NULL, PRIMARY KEY(id)) STRICT, WITHOUT ROWID; CREATE UNIQUE INDEX bad_index ON bad(id);", + "CREATE TABLE bad (id TEXT NOT NULL, PRIMARY KEY(id)) STRICT, WITHOUT ROWID; CREATE INDEX bad_index ON bad(lower(id));", + "CREATE TABLE bad (id TEXT NOT NULL, PRIMARY KEY(id)) STRICT, WITHOUT ROWID; CREATE INDEX bad_index ON bad(id) WHERE id<>'';", "CREATE TABLE bad (id INTEGER, payload BLOB) STRICT; CREATE TRIGGER bad_trigger AFTER INSERT ON bad BEGIN SELECT 1; END;", "CREATE VIEW bad AS SELECT 1;" }) @@ -96,6 +100,7 @@ public static async Task RunAsync() var empty = Profile(Path.Combine(root, "empty-" + Guid.NewGuid().ToString("N") + ".db")); CreateEmpty(empty.Database); Check(!(await LogicalMigrationService.PreviewAsync(blocked, empty, 3)).CanStart && await ObjectCountAsync(empty) == 0, "Catalog blocks unsupported SQLite semantics"); } + await TextKeysAsync(root); } finally { Directory.Delete(root, true); } } @@ -117,18 +122,27 @@ public static async Task PostgreSqlLiveAsync() await admin.ExecuteAsync("postgres", "CREATE DATABASE \"" + postgres.Database + "\";"); created = true; var sqlite = Profile(Path.Combine(root, "source.db")); using var origin = DatabaseProviderFactory.Create(sqlite); - await origin.ExecuteAsync(sqlite.Database, "CREATE TABLE sample (number INTEGER, payload BLOB NOT NULL) STRICT; INSERT INTO sample VALUES (-9223372036854775808,X'00FF'),(9223372036854775807,X''),(NULL,X'01'),(NULL,X'01');"); + await origin.ExecuteAsync(sqlite.Database, "CREATE TABLE sample (number INTEGER, payload BLOB NOT NULL) STRICT; INSERT INTO sample VALUES (-9223372036854775808,X'00FF'),(9223372036854775807,X''),(NULL,X'01'),(NULL,X'01');" + BusinessSql); var forward = await LogicalMigrationService.PreviewAsync(sqlite, postgres, 16); Check(forward.CanStart, "Real PostgreSQL catalog allows restricted empty target: " + string.Join(";", forward.Blockers)); Check((await LogicalMigrationService.RunAsync(sqlite, postgres, 16, forward, postgres.Database)).Outcome == LogicalMigrationOutcome.Verified, "SQLite -> PostgreSQL real typed transfer"); + var pgOrder = await admin.ExecuteAsync(postgres.Database, "SELECT label FROM customers ORDER BY label;"); + Check(pgOrder.Rows.Select(row => Convert.ToString(row[0])).SequenceEqual(new[] { "A", "a", "é", "臺灣😀" }), "Real PostgreSQL C collation retains binary Unicode ordering"); + try { await admin.ExecuteAsync(postgres.Database, "INSERT INTO lines VALUES(1,'A');"); throw new Exception("PG composite PK must reject duplicate"); } + catch (Npgsql.PostgresException error) { Check(error.SqlState == "23505", "Real PG primary key enforced"); } var reverse = Profile(Path.Combine(root, "target.db")); CreateEmpty(reverse.Database); var backward = await LogicalMigrationService.PreviewAsync(postgres, reverse, 3); - Check(backward.CanStart && backward.Tables.Single().ContentHash == forward.Tables.Single().ContentHash, "PostgreSQL -> SQLite real catalog and content"); + Check(backward.CanStart && backward.Tables.Select(table => table.ContentHash).SequenceEqual(forward.Tables.Select(table => table.ContentHash)), "PostgreSQL -> SQLite real catalog and content"); Check((await LogicalMigrationService.RunAsync(postgres, reverse, 3, backward, reverse.Database)).Outcome == LogicalMigrationOutcome.Verified, "PostgreSQL -> SQLite verified commit"); + await CheckBusinessAsync(reverse); using var pgOwner = DatabaseProviderFactory.Create(postgres); foreach (var mutation in new[] { + ("CREATE INDEX extra ON sample((number+1));", "DROP INDEX extra;"), ("CREATE INDEX extra ON sample(number);", "DROP INDEX extra;"), + ("CREATE UNIQUE INDEX extra ON customers(label);", "DROP INDEX extra;"), + ("CREATE INDEX extra ON customers(label COLLATE \"POSIX\");", "DROP INDEX extra;"), + ("ALTER TABLE customers ALTER COLUMN label TYPE text COLLATE \"default\";", "ALTER TABLE customers ALTER COLUMN label TYPE text COLLATE \"C\";"), ("ALTER TABLE sample ALTER COLUMN number SET DEFAULT 1;", "ALTER TABLE sample ALTER COLUMN number DROP DEFAULT;"), ("CREATE SEQUENCE extra_sequence;", "DROP SEQUENCE extra_sequence;"), ("SELECT lo_create(1736281931);", "SELECT lo_unlink(1736281931);"), @@ -154,6 +168,63 @@ public static async Task PostgreSqlLiveAsync() } } + private const string BusinessSql = "CREATE TABLE customers (id INTEGER NOT NULL, label TEXT COLLATE BINARY NOT NULL, note TEXT, payload BLOB, PRIMARY KEY(id)) STRICT, WITHOUT ROWID; " + + "CREATE INDEX customer_label ON customers(label); CREATE INDEX customer_note ON customers(note,id); " + + "CREATE TABLE lines (customer INTEGER NOT NULL, code TEXT NOT NULL, PRIMARY KEY(customer,code)) STRICT, WITHOUT ROWID; " + + "INSERT INTO customers VALUES (1,'臺灣😀','',X'00FF'),(2,'A',NULL,NULL),(3,'a','é',X''),(4,'é','中文',NULL); " + + "INSERT INTO lines VALUES (1,'A'),(1,'a'),(2,'😀');"; + + private static async Task TextKeysAsync(string root) + { + var source = Profile(Path.Combine(root, "business.db")); + var target = Profile(Path.Combine(root, "business-target.db")); CreateEmpty(target.Database); + using var owner = DatabaseProviderFactory.Create(source); + await owner.ExecuteAsync(source.Database, BusinessSql); + var preview = await LogicalMigrationService.PreviewAsync(source, target, 3); + Check(preview.CanStart && preview.Tables.Single(table => table.Name == "customers").PrimaryKey.SequenceEqual(new[] { "id" }) && preview.Statements.Count == 4, "Real text, manual integer/composite PK and ordinary indexes preview: " + string.Join(";", preview.Blockers)); + using (var cancel = new CancellationTokenSource()) + { + var canceled = await LogicalMigrationService.RunWithObserverAsync(source, target, 3, preview, target.Database, cancel.Token, + (stage, connection, transaction) => { if (stage == "index") cancel.Cancel(); }); + Check(canceled.Outcome == LogicalMigrationOutcome.RolledBack && await ObjectCountAsync(target) == 0, "Cancel after first real index rolls back schema, indexes and rows"); + } + await owner.ExecuteAsync(source.Database, "DROP INDEX customer_note; CREATE INDEX customer_note ON customers(id,note);"); + var changed = await LogicalMigrationService.RunAsync(source, target, 3, preview, target.Database); + Check(changed.Outcome == LogicalMigrationOutcome.RolledBack && await ObjectCountAsync(target) == 0, "Index column-order change invalidates approved fingerprint"); + await owner.ExecuteAsync(source.Database, "DROP INDEX customer_note; CREATE INDEX customer_note ON customers(note,id);"); + var failed = await LogicalMigrationService.RunWithObserverAsync(source, target, 3, preview, target.Database, CancellationToken.None, + (stage, connection, transaction) => + { + if (stage != "verify") return; + using var command = connection.CreateCommand(); command.Transaction = transaction; command.CommandText = "DROP INDEX customer_note;"; command.ExecuteNonQuery(); + }); + Check(failed.Outcome == LogicalMigrationOutcome.RolledBack && await ObjectCountAsync(target) == 0, "Missing index in verification rolls back all schema/data"); + Check((await LogicalMigrationService.RunAsync(source, target, 3, preview, target.Database)).Outcome == LogicalMigrationOutcome.Verified, "Real business schema/content verified transfer"); + await CheckBusinessAsync(target); + var empty = Profile(Path.Combine(root, "text-invalid-target.db")); CreateEmpty(empty.Database); + foreach (var value in new[] { "CAST(X'FF' AS TEXT)", "CAST(X'410042' AS TEXT)", "printf('%.*c',1048577,'a')" }) + { + await owner.ExecuteAsync(source.Database, "UPDATE customers SET note=" + value + " WHERE id=1;"); + try { await LogicalMigrationService.PreviewAsync(source, empty, 3); throw new Exception("Invalid or oversized text must be refused"); } + catch (InvalidOperationException) { Check(await ObjectCountAsync(empty) == 0, "Invalid UTF8/NUL/oversized text leaves empty target"); } + } + Console.WriteLine("PASS 真實 SQLite 文字、主鍵、索引、限額與回滾子案例"); + } + + private static async Task CheckBusinessAsync(ConnectionProfile profile) + { + using var owner = DatabaseProviderFactory.Create(profile); + var ordered = await owner.ExecuteAsync(profile.Database, "SELECT label FROM customers ORDER BY label;"); + Check(ordered.Rows.Select(row => Convert.ToString(row[0])).SequenceEqual(new[] { "A", "a", "é", "臺灣😀" }), "Binary UTF8 collation order preserved"); + var indexes = await owner.ExecuteAsync(profile.Database, "PRAGMA index_list(customers);"); + Check(indexes.Rows.Any(row => Equals(row[1], "customer_label")) && indexes.Rows.Any(row => Equals(row[1], "customer_note")), "Actual index names preserved"); + foreach (var sql in new[] { "INSERT INTO customers(id,label) VALUES (1,'duplicate');", "INSERT INTO customers(label) VALUES ('no automatic identity');", "INSERT INTO lines VALUES(1,'A');" }) + { + try { await owner.ExecuteAsync(profile.Database, sql); throw new Exception("PK constraint must reject write"); } + catch (SqliteException) { } + } + } + private static string Required(string key) => Environment.GetEnvironmentVariable(key) ?? throw new InvalidOperationException("Missing isolated fixture variable " + key); private static ConnectionProfile Profile(string path) => new() { Provider = DatabaseProviderKind.Sqlite, Name = "Isolated logical migration", Database = path }; private static void CreateEmpty(string path) { using var connection = new SqliteConnection("Data Source=" + path); connection.Open(); } diff --git a/mySQLPunk.Desktop/LogicalMigrationWindow.cs b/mySQLPunk.Desktop/LogicalMigrationWindow.cs index b870d6e..fb956bb 100644 --- a/mySQLPunk.Desktop/LogicalMigrationWindow.cs +++ b/mySQLPunk.Desktop/LogicalMigrationWindow.cs @@ -35,7 +35,7 @@ public LogicalMigrationWindow(ConnectionProfile source, IReadOnlyList profile.Name + " · " + profile.ProviderDisplayName).ToArray(); var fields = new StackPanel { Spacing = 8, Margin = new Avalonia.Thickness(16) }; fields.Children.Add(new TextBlock { Text = "來源：" + source.ProviderDisplayName + " / " + source.Database, TextWrapping = Avalonia.Media.TextWrapping.Wrap }); - fields.Children.Add(new TextBlock { Text = "不是原生備份 restore。此階段僅 SQLite STRICT ↔ PostgreSQL 14–17：INTEGER/bigint、BLOB/bytea、NULL/NOT NULL；其他型別、PK/FK、索引、default、identity/sequence、trigger/procedure、schema 相依一律預檢阻擋。目標須事先另建且為空。最多 100,000 列／64 MiB，單一交易＋內容校驗，不提供逐表續寫。", TextWrapping = Avalonia.Media.TextWrapping.Wrap }); + fields.Children.Add(new TextBlock { Text = "不是原生備份 restore。SQLite STRICT ↔ PostgreSQL 14–17：INTEGER/bigint、BLOB/bytea、有效 UTF-8 TEXT（BINARY ↔ C）、NULL/NOT NULL、非自增單欄／複合主鍵、普通升冪非 unique 索引。SQLite 主鍵表須 WITHOUT ROWID 且欄位明確 NOT NULL；nullable 索引採 NULLS FIRST。FK、其他定序、default、identity/sequence、expression/partial/unique 索引、trigger/procedure 仍預檢阻擋。目標須事先另建且為空。最多 100,000 列／64 MiB、單欄 1 MiB，單一交易＋結構與內容校驗，不逐表續寫。", TextWrapping = Avalonia.Media.TextWrapping.Wrap }); foreach (var control in new Control[] { _target, _database, _version, _password, _sshPassword, _passphrase, _confirmation }) fields.Children.Add(control); var buttons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 8 }; buttons.Children.Add(_inspect); buttons.Children.Add(_apply); buttons.Children.Add(_cancel); fields.Children.Add(buttons); @@ -58,7 +58,7 @@ public LogicalMigrationWindow(ConnectionProfile source, IReadOnlyList inspectMigration.IsEnabled && applyMigration.IsEnabled); Check(((TextBox)Field(migration, "_report")!).Text!.Contains("STRICT"), "Actual migration UI renders schema/data preview without row values"); + Check(((TextBox)Field(migration, "_report")!).Text!.Contains("CREATE INDEX") && !((TextBox)Field(migration, "_report")!).Text!.Contains("臺灣😀"), "Actual UI previews text/PK/index DDL without exposing values"); ((TextBox)Field(migration, "_database")!).Text = migrationTarget + ".changed"; Check(!applyMigration.IsEnabled, "Changed target invalidates UI approval"); ((TextBox)Field(migration, "_database")!).Text = migrationTarget; @@ -113,8 +114,13 @@ Check(!inspectMigration.IsEnabled && !applyMigration.IsEnabled, "Verified UI completion forbids duplicate resume"); using (var database = new SqliteConnection("Data Source=" + migrationTarget)) { - database.Open(); using var command = database.CreateCommand(); command.CommandText = "SELECT count(*) FROM fixture WHERE id=1 AND label=X'00FF';"; - Check(Convert.ToInt64(command.ExecuteScalar()) == 1, "Actual UI transfer preserves integer and binary data"); + database.Open(); using var command = database.CreateCommand(); command.CommandText = "SELECT count(*) FROM fixture WHERE id=1 AND label=X'00FF' AND name='臺灣😀';"; + Check(Convert.ToInt64(command.ExecuteScalar()) == 1, "Actual UI transfer preserves integer, text and binary data"); + command.CommandText = "SELECT count(*) FROM sqlite_schema WHERE name='fixture_name' AND type='index';"; + Check(Convert.ToInt64(command.ExecuteScalar()) == 1, "Actual UI transfer creates approved index"); + command.CommandText = "INSERT INTO fixture VALUES(1,NULL,'duplicate');"; + try { command.ExecuteNonQuery(); throw new Exception("Actual UI target PK must reject duplicate"); } + catch (SqliteException) { Check(true, "Actual UI transfer retains PK enforcement"); } } migration.Close(); await Wait(() => Field(window, "_operationCancellation") == null); From ccef849a51fa633cfffcadf11a6b11a40d3ff7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=94=E6=AF=94?= Date: Wed, 14 Oct 2026 13:03:20 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E8=A8=88=E5=85=A5=E7=A9=BA=E6=AC=84?= =?UTF-8?q?=E4=BD=8D=E9=A0=90=E7=AE=97=E4=B8=A6=E9=A9=97=E8=AD=89=20Postgr?= =?UTF-8?q?eSQL=20=E6=AC=84=E4=BD=8D=E9=99=90=E9=A1=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NULL 與空欄位也佔快照空間，至少計入八個位元組，避免大量空值繞過總量限制。加入 PostgreSQL 真實 fixture 的 32 MiB 拒絕與 1 MiB 邊界搬移，確認大欄位不會先完整傳回應用程式。 不支援的 expression 索引測試改用取餘數，避免 Int64 最大值在建立測試索引時先溢位，讓測試確實走到 catalog 預檢。 --- docs/LOGICAL_MIGRATION_MATRIX.md | 8 ++++---- mySQLPunk.Core/Services/LogicalMigrationService.cs | 4 ++-- .../LogicalMigrationTests.cs | 12 +++++++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/LOGICAL_MIGRATION_MATRIX.md b/docs/LOGICAL_MIGRATION_MATRIX.md index d867376..9735445 100644 --- a/docs/LOGICAL_MIGRATION_MATRIX.md +++ b/docs/LOGICAL_MIGRATION_MATRIX.md @@ -18,9 +18,9 @@ - SQLite 文字只接受 BINARY（隱含或明確），PostgreSQL 必須是內建 `pg_catalog."C"`；兩端 UTF-8，逐欄嚴格解碼，拒絕 NUL、無效 UTF-8，不偷偷替換字元。保留空字串、NULL、Unicode、大小寫與 byte-order 比較語意；不承諾兩引擎字串函式或 LIKE 行為相同。 - SQLite PK 表要求完整可辨識 grammar、明確 NOT NULL 與 WITHOUT ROWID，避免 INTEGER PRIMARY KEY 的 rowid alias 自動給 ID 行為被偷偷改成 PostgreSQL 無 sequence 的 bigint。單欄整數／文字／二進位與複合 PK 均可；不保證引擎自動產生的 PK constraint／內部索引名稱或實體配置相同。 -- 普通非 unique btree 索引保留名稱、欄位與順序，只允許升冪、預設內建 opclass、無 expression／partial／INCLUDE／自訂 storage。SQLite 升冪 NULL 在前，目標 PostgreSQL 明確 `ASC NULLS FIRST`；反向 nullable 欄位索引若為 PostgreSQL 預設 NULLS LAST 則拒絕，NOT NULL 索引沒有這項差异。DESC、unique（PK 除外）、自訂定序／opclass、clustered／replica identity 索引都阻擋。 +- 普通非 unique btree 索引保留名稱、欄位與順序，只允許升冪、預設內建 opclass、無 expression／partial／INCLUDE／自訂 storage。SQLite 升冪 NULL 在前，目標 PostgreSQL 明確 `ASC NULLS FIRST`；反向 nullable 欄位索引若為 PostgreSQL 預設 NULLS LAST 則拒絕，NOT NULL 索引沒有這項差異。DESC、unique（PK 除外）、自訂定序／opclass、clustered／replica identity 索引都阻擋。 - SQLite／PostgreSQL schema＋索引＋資料仍為單一交易，不引入逐表 partial success。每次重新預檢／最終驗證涵蓋 PK、索引、內容；已提交或 ack lost 不可盲重試。 -- 有界讀取先在 SQL 投影取 byte length、超限 CASE 回傳 NULL，再检查預算，才 materialize 欄位。BLOB 不會先 GetValue 完整大陣列；SQLite expression 用已限額的 byte[] 讀取（避開 provider GetBytes 的 metadata 路徑），PostgreSQL 分段讀取。這是應用端快照限額，不是對引擎自身 page cache／SQL 執行記憶體的硬上限。 +- 有界讀取先在 SQL 投影取 byte length、超限 CASE 回傳 NULL，再檢查預算，才 materialize 欄位。BLOB 不會先 GetValue 完整大陣列；SQLite expression 用已限額的 byte[] 讀取（避開 provider GetBytes 的 metadata 路徑），PostgreSQL 分段讀取。NULL 與空欄位至少計入 8 bytes 預算。這是應用端快照限額，不是對引擎自身 page cache／SQL 執行記憶體的硬上限。 SQLite／PostgreSQL 工具入口在 Avalonia 工作台「邏輯搬移」。來源使用目前連線的記憶體設定；目標選已儲存的連線及另建空資料庫，明確選版本，必要密碼只由使用者輸入，不額外讀秘密庫。獨立 provider owner 保有 SSH tunnel，所有開啟／失敗／結束都有 scope 釋放；不從 redacted ConnectionString 重建。 @@ -45,7 +45,7 @@ SQLite／PostgreSQL 工具入口在 Avalonia 工作台「邏輯搬移」。來 | MySQL/MariaDB → SQLite/PG/SQLServer | 整數 unsigned 上界、zero date、ENUM/SET、charset、PAD SPACE 定序、AUTO_INCREMENT 與 storage engine 一致快照不能用現有 count-only transfer 當忠實轉換 | 來源 catalog／版本／InnoDB 快照 adapter，逐欄區間及 charset/定序證明；不可先沿用一般 type map 再補驗證 | | SQLite/PG/SQLServer → MySQL/MariaDB，以及 MySQL↔MariaDB／跨版本 | MySQL DDL implicit commit，DDL＋資料單一交易會是假保證；目標即使原本空白，取消仍可能已留下表 | 另建 staging 與 durable uncertain/checkpoint 協議、預檢引擎版本差異、人工核對恢復；不能自動刪 staging 或覆蓋目標假裝 rollback | | SQLServer → SQLite/PG/MySQL，以及反向 | UTF-16/UTF-8、linguistic/BIN2 定序的比較／尾空白規則、rowversion、decimal/time scale、identity/default/sequence 不能把現有型別名稱替換當成保留語意 | 可重用 SQLServer catalog 的阻擋規則，但需跨 provider canonical schema／資料 codec、來源一致快照與逐欄語意／版本證據；目前只 Windows SQLServer→SQLServer 專用子集可執行 | -| Oracle/Snowflake 與其他引擎 | NUMBER／時間、空字串 NULL、identifier/collation 與交易 DDL 不等价 | 專用 catalog／codec／交易恢復設計；本輪不開連線 | +| Oracle/Snowflake 與其他引擎 | NUMBER／時間、空字串 NULL、identifier/collation 與交易 DDL 不等價 | 專用 catalog／codec／交易恢復設計；本輪不開連線 | | MongoDB、Redis/Garnet 與表格引擎 | document/key-value model、TTL／原子操作／巢狀內容沒有通用無損表格映射 | 明確專用模型轉換契約，不當成同義 restore | 上述方向服務在開連線前拒絕，沒有聲稱某個 MySQL/MariaDB 或 SQLServer 版本已跨引擎通過。雙向跨版本／換引擎的整體需求仍未全部完成；此批是實際擴充 SQLite/PG 的可驗證第二階段，而不是全引擎完成或發版的宣告。 @@ -56,7 +56,7 @@ SQLite／PostgreSQL 工具入口在 Avalonia 工作台「邏輯搬移」。來 實際隔離 SQLite：1,005 列、多批、Int64 極值、空／非空 bytes、NULL、重複列、反向內容一致、錯誤確認、缺目標不自動建立、來源同列數內容改變、unsupported features、部分寫入失敗、內容／結構驗證失敗、途中取消，以及真正 commit 後注入 ack lost（保留資料且禁止重試）。不是單靠手動設 mock Blockers。 -本輪新增正式 SQLite：32 MiB BLOB 在 materialize 前拒絕且 managed allocation 小於 8 MiB、1 MiB 邊界搬移、有效 Unicode／空字串／NULL／NUL／無效 UTF-8／文字超限、單欄與複合 PK 真的拒絕重複及缺 ID、索引缺失／順序變动與索引建立後取消整批回滾。PostgreSQL 16 CI fixture 用同一業務表雙向搬移與 C 定序／複合 PK 實際檢查，另對 expression/unique/null ordering/collation 真 catalog 加入與移除後預檢。 +本輪新增正式 SQLite：32 MiB BLOB 在 materialize 前拒絕且 managed allocation 小於 8 MiB、1 MiB 邊界搬移、有效 Unicode／空字串／NULL／NUL／無效 UTF-8／文字超限、單欄與複合 PK 真的拒絕重複及缺 ID、索引缺失／順序變動與索引建立後取消整批回滾。PostgreSQL 16 CI fixture 用同一業務表雙向搬移與 C 定序／複合 PK 實際檢查，另對 expression/unique/null ordering/collation 真 catalog 加入與移除後預檢，以及 32 MiB bytea 有界拒絕／1 MiB 邊界搬移。 Avalonia/Xvfb：正式入口持有主視窗 gate、預覽、目標變更失效、錯誤確認不寫入、實際確認寫入且禁止重複 resume。既有跨平台 CI fixture 另測 SQLite ↔ PostgreSQL 16，包括正式 catalog 逐項加 feature／移除 feature 後恢復可預檢。 diff --git a/mySQLPunk.Core/Services/LogicalMigrationService.cs b/mySQLPunk.Core/Services/LogicalMigrationService.cs index d43b319..bc88754 100644 --- a/mySQLPunk.Core/Services/LogicalMigrationService.cs +++ b/mySQLPunk.Core/Services/LogicalMigrationService.cs @@ -269,8 +269,8 @@ private static async Task ReadSnapshotAsync(DbConnection connection, D for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) { var binary = columns[columnIndex].Kind != "integer"; - var bytes = reader.IsDBNull(ordinal) ? 0 : binary ? reader.GetInt64(ordinal) : 8; - totalBytes += bytes; + var bytes = reader.IsDBNull(ordinal) ? 8 : binary ? reader.GetInt64(ordinal) : 8; + totalBytes += Math.Max(8, bytes); if (bytes > MaximumFieldBytes || totalBytes > MaximumBytes) throw new InvalidOperationException("超過單欄 1 MiB／總量 64 MiB 預覽上限。"); if (binary) ordinal++; object? value = reader.IsDBNull(ordinal) ? null : binary ? ReadBoundedBytes(reader, ordinal, bytes, token) : reader.GetInt64(ordinal); diff --git a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs index 693c770..f729fdc 100644 --- a/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs +++ b/mySQLPunk.CrossPlatform.SmokeTests/LogicalMigrationTests.cs @@ -130,6 +130,16 @@ public static async Task PostgreSqlLiveAsync() Check(pgOrder.Rows.Select(row => Convert.ToString(row[0])).SequenceEqual(new[] { "A", "a", "é", "臺灣😀" }), "Real PostgreSQL C collation retains binary Unicode ordering"); try { await admin.ExecuteAsync(postgres.Database, "INSERT INTO lines VALUES(1,'A');"); throw new Exception("PG composite PK must reject duplicate"); } catch (Npgsql.PostgresException error) { Check(error.SqlState == "23505", "Real PG primary key enforced"); } + var oversizedTarget = Profile(Path.Combine(root, "pg-oversized-target.db")); CreateEmpty(oversizedTarget.Database); + await admin.ExecuteAsync(postgres.Database, "CREATE TABLE bounded_payload(payload bytea); INSERT INTO bounded_payload VALUES(decode(repeat('FF',33554432),'hex')); "); + var allocatedBefore = GC.GetTotalAllocatedBytes(true); + try { await LogicalMigrationService.PreviewAsync(postgres, oversizedTarget, 3); throw new Exception("Oversize PG BLOB must be refused"); } + catch (InvalidOperationException error) { Check(error.Message.Contains("1 MiB"), "PG length refusal precedes streaming materialization"); } + Check(GC.GetTotalAllocatedBytes(true) - allocatedBefore < 8 * 1024 * 1024 && await ObjectCountAsync(oversizedTarget) == 0, "Real PG 32 MiB BLOB is not transported as a full managed field and target stays empty"); + await admin.ExecuteAsync(postgres.Database, "UPDATE bounded_payload SET payload=decode(repeat('FF',1048576),'hex');"); + var boundary = await LogicalMigrationService.PreviewAsync(postgres, oversizedTarget, 3); + Check(boundary.CanStart && (await LogicalMigrationService.RunAsync(postgres, oversizedTarget, 3, boundary, oversizedTarget.Database)).Outcome == LogicalMigrationOutcome.Verified, "Real PG 1 MiB boundary streams and verifies into SQLite"); + await admin.ExecuteAsync(postgres.Database, "DROP TABLE bounded_payload;"); var reverse = Profile(Path.Combine(root, "target.db")); CreateEmpty(reverse.Database); var backward = await LogicalMigrationService.PreviewAsync(postgres, reverse, 3); Check(backward.CanStart && backward.Tables.Select(table => table.ContentHash).SequenceEqual(forward.Tables.Select(table => table.ContentHash)), "PostgreSQL -> SQLite real catalog and content"); @@ -138,7 +148,7 @@ public static async Task PostgreSqlLiveAsync() using var pgOwner = DatabaseProviderFactory.Create(postgres); foreach (var mutation in new[] { - ("CREATE INDEX extra ON sample((number+1));", "DROP INDEX extra;"), + ("CREATE INDEX extra ON sample((number%2));", "DROP INDEX extra;"), ("CREATE INDEX extra ON sample(number);", "DROP INDEX extra;"), ("CREATE UNIQUE INDEX extra ON customers(label);", "DROP INDEX extra;"), ("CREATE INDEX extra ON customers(label COLLATE \"POSIX\");", "DROP INDEX extra;"),