diff --git a/Cargo.lock b/Cargo.lock index 212bf9a..0269bff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -86,7 +97,7 @@ dependencies = [ [[package]] name = "aingle_ai" -version = "0.4.2" +version = "0.5.0" dependencies = [ "blake2", "candle-core 0.9.2", @@ -108,7 +119,7 @@ dependencies = [ [[package]] name = "aingle_contracts" -version = "0.4.2" +version = "0.5.0" dependencies = [ "blake3", "dashmap 6.1.0", @@ -127,10 +138,12 @@ dependencies = [ [[package]] name = "aingle_cortex" -version = "0.4.2" +version = "0.5.0" dependencies = [ "aingle_graph", "aingle_logic", + "aingle_raft", + "aingle_wal", "aingle_zk", "argon2", "async-graphql", @@ -149,18 +162,22 @@ dependencies = [ "log", "mdns-sd", "once_cell", + "openraft", "quinn", "rand 0.9.2", "rcgen", "regex", "reqwest", "rustls", + "rustls-pemfile", "serde", "serde_json", "spargebra", + "subtle", "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls", "tokio-stream", "tokio-test", "tower", @@ -173,7 +190,7 @@ dependencies = [ [[package]] name = "aingle_graph" -version = "0.4.2" +version = "0.5.0" dependencies = [ "bincode", "blake3", @@ -190,11 +207,12 @@ dependencies = [ "sled", "tempfile", "thiserror 2.0.18", + "uuid", ] [[package]] name = "aingle_logic" -version = "0.4.2" +version = "0.5.0" dependencies = [ "aingle_graph", "chrono", @@ -210,7 +228,7 @@ dependencies = [ [[package]] name = "aingle_minimal" -version = "0.4.2" +version = "0.5.0" dependencies = [ "async-io", "async-tungstenite", @@ -250,9 +268,30 @@ dependencies = [ "webrtc", ] +[[package]] +name = "aingle_raft" +version = "0.5.0" +dependencies = [ + "aingle_graph", + "aingle_wal", + "anyerror", + "bincode", + "blake3", + "chrono", + "futures-util", + "ineru", + "openraft", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-test", + "tracing", +] + [[package]] name = "aingle_viz" -version = "0.4.2" +version = "0.5.0" dependencies = [ "aingle_graph", "aingle_minimal", @@ -272,9 +311,21 @@ dependencies = [ "uuid", ] +[[package]] +name = "aingle_wal" +version = "0.5.0" +dependencies = [ + "bincode", + "blake3", + "chrono", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "aingle_zk" -version = "0.4.2" +version = "0.5.0" dependencies = [ "blake3", "bulletproofs", @@ -372,6 +423,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyerror" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71add24cc141a1e8326f249b74c41cfd217aeb2a67c9c6cf9134d175469afd49" +dependencies = [ + "serde", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -1016,6 +1076,18 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.6" @@ -1117,6 +1189,29 @@ dependencies = [ "dbus", ] +[[package]] +name = "borsh" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "bstr" version = "1.12.1" @@ -1198,18 +1293,52 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "byte-unit" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c6d47a4e2961fb8721bcfc54feae6455f2f64e7054f9bc67e875f0e77f4c58d" +dependencies = [ + "rust_decimal", + "schemars", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive 0.6.12", + "ptr_meta 0.1.4", + "simdutf8", +] + [[package]] name = "bytecheck" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" dependencies = [ - "bytecheck_derive", - "ptr_meta", + "bytecheck_derive 0.8.2", + "ptr_meta 0.3.1", "rancor", "simdutf8", ] +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytecheck_derive" version = "0.8.2" @@ -2399,7 +2528,7 @@ dependencies = [ "aes", "aes-gcm", "async-trait", - "bytecheck", + "bytecheck 0.8.2", "byteorder", "cbc", "ccm", @@ -2414,7 +2543,7 @@ dependencies = [ "rand_core 0.6.4", "rcgen", "ring", - "rkyv", + "rkyv 0.8.15", "rustls", "sec1", "sha1", @@ -2426,6 +2555,12 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "dyn-stack" version = "0.10.0" @@ -3019,6 +3154,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.32" @@ -3534,13 +3675,22 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.12", ] [[package]] @@ -3977,7 +4127,7 @@ dependencies = [ [[package]] name = "ineru" -version = "0.4.2" +version = "0.5.0" dependencies = [ "bincode", "blake3", @@ -4236,7 +4386,7 @@ dependencies = [ [[package]] name = "kaneru" -version = "0.4.2" +version = "0.5.0" dependencies = [ "chrono", "criterion", @@ -4499,6 +4649,12 @@ dependencies = [ "zerocopy-derive", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -5002,6 +5158,67 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openraft" +version = "0.10.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b9d8db10f834d517e4c2c45ab5c645bc5cafee9d07f7b150b8029a0b1ebdca" +dependencies = [ + "anyerror", + "byte-unit", + "chrono", + "clap", + "derive_more", + "futures-util", + "maplit", + "openraft-macros", + "openraft-rt", + "openraft-rt-tokio", + "peel-off", + "rand 0.9.2", + "serde", + "thiserror 2.0.18", + "tracing", + "validit", +] + +[[package]] +name = "openraft-macros" +version = "0.10.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22b0bd215948ed47997a1d0447ea592e49220096360a833b118f329a08aa286" +dependencies = [ + "chrono", + "proc-macro2", + "quote", + "semver", + "syn 2.0.117", +] + +[[package]] +name = "openraft-rt" +version = "0.10.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b651e6e2f25d022e34549e605eb8875c78ebc26862b16b06143a551e53ec00" +dependencies = [ + "futures-channel", + "futures-util", + "openraft-macros", + "rand 0.9.2", +] + +[[package]] +name = "openraft-rt-tokio" +version = "0.10.0-alpha.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478d5625fdeb13293e68549ba1d42b7a25085f3be04204412147637ad22e2827" +dependencies = [ + "futures-util", + "openraft-rt", + "rand 0.9.2", + "tokio", +] + [[package]] name = "openssl" version = "0.10.75" @@ -5174,6 +5391,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "peel-off" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3420ea4424090cbd75a688996f696a807c68d6744b4863591b86435dc3078e9" + [[package]] name = "peg" version = "0.8.5" @@ -5455,13 +5678,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive 0.1.4", +] + [[package]] name = "ptr_meta" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.3.1", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -5588,13 +5831,19 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rancor" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" dependencies = [ - "ptr_meta", + "ptr_meta 0.3.1", ] [[package]] @@ -5784,6 +6033,26 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "regalloc2" version = "0.13.5" @@ -5853,13 +6122,22 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck 0.6.12", +] + [[package]] name = "rend" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" dependencies = [ - "bytecheck", + "bytecheck 0.8.2", ] [[package]] @@ -5947,25 +6225,54 @@ dependencies = [ "rio_api", ] +[[package]] +name = "rkyv" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +dependencies = [ + "bitvec", + "bytecheck 0.6.12", + "bytes", + "hashbrown 0.12.3", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.46", + "seahash", + "tinyvec", + "uuid", +] + [[package]] name = "rkyv" version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a30e631b7f4a03dee9056b8ef6982e8ba371dd5bedb74d3ec86df4499132c70" dependencies = [ - "bytecheck", + "bytecheck 0.8.2", "bytes", "hashbrown 0.16.1", "indexmap", "munge", - "ptr_meta", + "ptr_meta 0.3.1", "rancor", - "rend", - "rkyv_derive", + "rend 0.5.3", + "rkyv_derive 0.8.15", "tinyvec", "uuid", ] +[[package]] +name = "rkyv_derive" +version = "0.7.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rkyv_derive" version = "0.8.15" @@ -6047,6 +6354,22 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust_decimal" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.5", + "rkyv 0.7.46", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -6135,6 +6458,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -6243,6 +6575,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -6261,6 +6605,12 @@ dependencies = [ "url", ] +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.7.3" @@ -6904,6 +7254,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.44" @@ -7496,6 +7852,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -7550,6 +7912,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "validit" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efba0434d5a0a62d4f22070b44ce055dc18cb64d4fa98276aa523dadfaba0e7" +dependencies = [ + "anyerror", +] + [[package]] name = "valuable" version = "0.1.1" @@ -7771,7 +8142,7 @@ dependencies = [ "object 0.38.1", "rangemap", "region", - "rkyv", + "rkyv 0.8.15", "self_cell", "shared-buffer", "smallvec", @@ -7825,14 +8196,14 @@ version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a7f91b0cb63705afa0843b46a0aeaeaedff7be2e5b05691176e9e58e2dbe921" dependencies = [ - "bytecheck", + "bytecheck 0.8.2", "enum-iterator", "enumset", "getrandom 0.2.17", "hex", "indexmap", "more-asserts", - "rkyv", + "rkyv 0.8.15", "sha2 0.11.0-rc.5", "target-lexicon", "thiserror 2.0.18", @@ -8641,6 +9012,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index 16f2619..672ec8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ members = [ "crates/aingle_minimal", # IoT-optimized minimal node "crates/aingle_contracts", # Smart Contracts (DSL + WASM Runtime) "crates/aingle_viz", # DAG Visualization Server + "crates/aingle_wal", # Write-Ahead Log (clustering) + "crates/aingle_raft", # Raft consensus (clustering) # ── Examples ──────────────────────────────────────────────────── "examples/iot_sensor_network", diff --git a/README.md b/README.md index 743871f..3941129 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@

Build Status License - Rust + Rust + Powers Mayros AI

@@ -156,6 +157,73 @@ Interactive D3.js dashboard. Watch your DAG evolve in real-time. Filter, search, --- +## Clustering + +AIngle supports multi-node clustering via Raft consensus for high availability and horizontal scalability. Writes are replicated to all nodes; reads can be served from any node with optional quorum consistency. + +### Quick Start (3-node cluster) + +```bash +# Node 1 — bootstrap leader +aingle-cortex --port 8081 \ + --cluster --cluster-node-id 1 \ + --cluster-secret "your-secret-at-least-16-chars" \ + --cluster-wal-dir ./data/node1/wal \ + --db-path ./data/node1/graph.sled + +# Node 2 — joins via node 1 +aingle-cortex --port 8082 \ + --cluster --cluster-node-id 2 \ + --cluster-peers 127.0.0.1:8081 \ + --cluster-secret "your-secret-at-least-16-chars" \ + --cluster-wal-dir ./data/node2/wal \ + --db-path ./data/node2/graph.sled + +# Node 3 — joins via node 1 +aingle-cortex --port 8083 \ + --cluster --cluster-node-id 3 \ + --cluster-peers 127.0.0.1:8081 \ + --cluster-secret "your-secret-at-least-16-chars" \ + --cluster-wal-dir ./data/node3/wal \ + --db-path ./data/node3/graph.sled +``` + +### With TLS encryption + +```bash +# Auto-generated self-signed certs (development) +aingle-cortex --port 8081 --cluster --cluster-node-id 1 \ + --cluster-secret "your-secret" --cluster-tls + +# Custom certificates (production) +aingle-cortex --port 8081 --cluster --cluster-node-id 1 \ + --cluster-secret "your-secret" --cluster-tls \ + --cluster-tls-cert /path/to/cert.pem \ + --cluster-tls-key /path/to/key.pem +``` + +### Cluster endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/v1/cluster/status` | GET | Node role, leader ID, current term | +| `/api/v1/cluster/members` | GET | All cluster members and their state | +| `/api/v1/cluster/join` | POST | Add a new node to the cluster | +| `/api/v1/cluster/leave` | POST | Gracefully remove a node | +| `/api/v1/cluster/wal/stats` | GET | WAL segment count and disk usage | +| `/api/v1/cluster/wal/verify` | POST | Verify WAL integrity (checksums) | + +### Features + +- **Raft consensus** — automatic leader election, log replication, and membership changes +- **Streaming snapshots** — 512KB chunked transfer with per-chunk ACK for large datasets +- **Write-Ahead Log** — crash-safe durability with segment rotation and integrity verification +- **TLS encryption** — optional TLS for inter-node communication (self-signed or custom certs) +- **Constant-time auth** — cluster secret verified with timing-safe comparison +- **Quorum reads** — optional strong consistency for read operations + +--- + ## Architecture ``` @@ -177,6 +245,12 @@ Interactive D3.js dashboard. Watch your DAG evolve in real-time. Filter, search, │ │ Graph │ │ Engine │ │ (Privacy) │ │ Runtime │ │ │ └──────────────┘ └──────────────┘ └──────────────┘ └───────────┘ │ ├────────────────────────────────────────────────────────────────────────┤ +│ CONSENSUS LAYER │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌───────────┐ │ +│ │ Raft │ │ WAL │ │ Streaming │ │ TLS │ │ +│ │ (openraft) │ │ (Durability) │ │ Snapshots │ │ (mTLS) │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └───────────┘ │ +├────────────────────────────────────────────────────────────────────────┤ │ NETWORK LAYER │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌───────────┐ │ │ │ Kitsune P2P │ │ CoAP │ │ Gossip │ │ mDNS │ │ @@ -199,6 +273,9 @@ cd aingle # Build cargo build --workspace --release +# Build with clustering support +cargo build -p aingle_cortex --features cluster --release + # Test cargo test --workspace @@ -208,7 +285,7 @@ cargo doc --workspace --no-deps --open ### Prerequisites -- **Rust** 1.70 or later +- **Rust** 1.83 or later - **libsodium-dev** (cryptography) - **libssl-dev** (TLS) - **pkg-config** @@ -264,6 +341,13 @@ cargo doc --workspace --no-deps --open | `aingle_logic` | Prolog-style reasoning engine | | `aingle_graph` | Semantic graph database | +### Clustering & Consensus + +| Component | Purpose | +|-----------|---------| +| `aingle_raft` | Raft consensus (leader election, log replication, streaming snapshots) | +| `aingle_wal` | Write-Ahead Log for crash-safe durability | + ### Security & Privacy | Component | Purpose | diff --git a/crates/aingle_ai/Cargo.toml b/crates/aingle_ai/Cargo.toml index 48095ae..3dd1334 100644 --- a/crates/aingle_ai/Cargo.toml +++ b/crates/aingle_ai/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_ai" -version = "0.4.2" +version = "0.5.0" description = "AI integration layer for AIngle - Ineru, Nested Learning, Kaneru" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" diff --git a/crates/aingle_contracts/Cargo.toml b/crates/aingle_contracts/Cargo.toml index 3c9f735..1d7a53a 100644 --- a/crates/aingle_contracts/Cargo.toml +++ b/crates/aingle_contracts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_contracts" -version = "0.4.2" +version = "0.5.0" description = "Smart Contracts DSL and WASM Runtime for AIngle" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" diff --git a/crates/aingle_cortex/Cargo.toml b/crates/aingle_cortex/Cargo.toml index 03693b3..677c41f 100644 --- a/crates/aingle_cortex/Cargo.toml +++ b/crates/aingle_cortex/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_cortex" -version = "0.4.2" +version = "0.5.0" description = "Córtex API - REST/GraphQL/SPARQL interface for AIngle semantic graphs" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -20,6 +20,7 @@ sparql = ["dep:spargebra"] auth = ["dep:jsonwebtoken", "dep:argon2"] p2p = ["dep:quinn", "dep:rustls", "dep:rcgen", "dep:ed25519-dalek", "dep:hex"] p2p-mdns = ["p2p", "dep:mdns-sd", "dep:if-addrs"] +cluster = ["p2p", "dep:aingle_wal", "dep:aingle_raft", "dep:openraft", "dep:tokio-rustls", "dep:rustls-pemfile"] full = ["rest", "graphql", "sparql", "auth"] [[bin]] @@ -28,10 +29,10 @@ path = "src/main.rs" [dependencies] # Core AIngle crates -aingle_graph = { version = "0.4", path = "../aingle_graph", features = ["sled-backend"] } -aingle_logic = { version = "0.4", path = "../aingle_logic" } -aingle_zk = { version = "0.4", path = "../aingle_zk" } -ineru = { version = "0.4", path = "../ineru" } +aingle_graph = { version = "0.5", path = "../aingle_graph", features = ["sled-backend"] } +aingle_logic = { version = "0.5", path = "../aingle_logic" } +aingle_zk = { version = "0.5", path = "../aingle_zk" } +ineru = { version = "0.5", path = "../ineru" } # Web framework axum = { version = "0.8", features = ["ws", "macros"] } @@ -68,6 +69,7 @@ rand = "0.9" # Hashing blake3 = "1.8" +subtle = "2.6" # Streaming tokio-stream = { version = "0.1", features = ["sync"] } @@ -92,6 +94,13 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std" rcgen = { version = "0.13", optional = true } ed25519-dalek = { version = "2", features = ["rand_core"], optional = true } hex = { version = "0.4", optional = true } +# Clustering (optional) +aingle_wal = { version = "0.5", path = "../aingle_wal", optional = true } +aingle_raft = { version = "0.5", path = "../aingle_raft", optional = true } +openraft = { version = "0.10.0-alpha.17", features = ["serde", "type-alias"], optional = true } +tokio-rustls = { version = "0.26", default-features = false, features = ["ring"], optional = true } +rustls-pemfile = { version = "2", optional = true } + dirs = "6" mdns-sd = { version = "0.18", optional = true } if-addrs = { version = "0.13", optional = true } diff --git a/crates/aingle_cortex/src/cluster_init.rs b/crates/aingle_cortex/src/cluster_init.rs new file mode 100644 index 0000000..17834aa --- /dev/null +++ b/crates/aingle_cortex/src/cluster_init.rs @@ -0,0 +1,534 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Cluster initialization — public API for setting up Raft consensus. +//! +//! This module extracts the cluster setup logic from `main.rs` into a +//! reusable API so it can be called both from the binary and from +//! integration tests. + +#[cfg(feature = "cluster")] +use crate::error::Error; +#[cfg(feature = "cluster")] +use crate::server::CortexServer; + +/// Configuration for cluster mode. +#[cfg(feature = "cluster")] +#[derive(Debug, Clone)] +pub struct ClusterConfig { + /// Whether cluster mode is enabled. + pub enabled: bool, + /// Unique Raft node ID (must be > 0). + pub node_id: u64, + /// Peer REST addresses to join (empty = bootstrap single-node). + pub peers: Vec, + /// Directory for the Write-Ahead Log. + pub wal_dir: Option, + /// Shared secret for authenticating internal cluster RPCs. + pub secret: Option, + /// Whether to use TLS for inter-node communication. + pub tls: bool, + /// Path to TLS certificate PEM file (optional; auto-generated if absent). + pub tls_cert: Option, + /// Path to TLS private key PEM file (optional; auto-generated if absent). + pub tls_key: Option, +} + +#[cfg(feature = "cluster")] +impl ClusterConfig { + /// Parse cluster config from CLI arguments. + pub fn from_args(args: &[String]) -> Self { + let mut cfg = Self { + enabled: false, + node_id: 0, + peers: Vec::new(), + wal_dir: None, + secret: None, + tls: false, + tls_cert: None, + tls_key: None, + }; + let mut i = 1; + while i < args.len() { + match args[i].as_str() { + "--cluster" => cfg.enabled = true, + "--cluster-node-id" => { + if i + 1 < args.len() { + cfg.node_id = args[i + 1].parse().unwrap_or(0); + i += 1; + } + } + "--cluster-peers" => { + if i + 1 < args.len() { + cfg.peers = + args[i + 1].split(',').map(|s| s.trim().to_string()).collect(); + i += 1; + } + } + "--cluster-wal-dir" => { + if i + 1 < args.len() { + cfg.wal_dir = Some(args[i + 1].clone()); + i += 1; + } + } + "--cluster-secret" => { + if i + 1 < args.len() { + cfg.secret = Some(args[i + 1].clone()); + i += 1; + } + } + "--cluster-tls" => cfg.tls = true, + "--cluster-tls-cert" => { + if i + 1 < args.len() { + cfg.tls_cert = Some(args[i + 1].clone()); + i += 1; + } + } + "--cluster-tls-key" => { + if i + 1 < args.len() { + cfg.tls_key = Some(args[i + 1].clone()); + i += 1; + } + } + _ => {} + } + i += 1; + } + cfg + } + + /// Validate the cluster configuration. Returns an error message on failure. + pub fn validate(&self) -> Result<(), String> { + if self.node_id == 0 { + return Err("--cluster-node-id must be > 0".into()); + } + if let Some(ref secret) = self.secret { + if secret.len() < 16 { + return Err("--cluster-secret must be at least 16 bytes".into()); + } + } + Ok(()) + } +} + +/// HTTP-based Raft RPC sender with exponential backoff. +/// +/// Routes Raft protocol messages to target nodes via their internal HTTP +/// endpoints (`/internal/raft/{append-entries,vote,snapshot}`). +#[cfg(feature = "cluster")] +pub struct HttpRaftRpcSender { + client: reqwest::Client, + cluster_secret: Option, + use_tls: bool, +} + +#[cfg(feature = "cluster")] +impl HttpRaftRpcSender { + /// Create a new sender. + /// + /// When `use_tls` is true, URLs will use `https://` and the reqwest + /// client will accept self-signed certificates (TOFU model, matching + /// the P2P transport). + pub fn new(cluster_secret: Option, use_tls: bool) -> Self { + let client = if use_tls { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .danger_accept_invalid_certs(true) // TOFU — same as P2P layer + .build() + .expect("Failed to create HTTPS client for Raft RPC") + } else { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build() + .expect("Failed to create HTTP client for Raft RPC") + }; + Self { + client, + cluster_secret, + use_tls, + } + } + + fn scheme(&self) -> &str { + if self.use_tls { + "https" + } else { + "http" + } + } +} + +#[cfg(feature = "cluster")] +impl aingle_raft::network::RaftRpcSender for HttpRaftRpcSender { + fn send_rpc( + &self, + addr: std::net::SocketAddr, + msg: aingle_raft::network::RaftMessage, + ) -> std::pin::Pin< + Box< + dyn std::future::Future> + + Send + + '_, + >, + > { + use aingle_raft::network::RaftMessage; + + Box::pin(async move { + let (path, payload) = match msg { + RaftMessage::AppendEntries { payload } => ("append-entries", payload), + RaftMessage::Vote { payload } => ("vote", payload), + RaftMessage::InstallSnapshot { payload } => ("snapshot", payload), + // Streaming snapshot chunks are routed to the chunk endpoint + ref chunk @ RaftMessage::SnapshotChunk { .. } => { + let payload = serde_json::to_vec(&chunk) + .map_err(|e| format!("Serialize snapshot chunk: {e}"))?; + ("snapshot-chunk", payload) + } + other => { + return Err(format!( + "Unsupported RaftMessage variant for HTTP RPC: {:?}", + std::mem::discriminant(&other) + )) + } + }; + + let url = format!("{}://{}/internal/raft/{}", self.scheme(), addr, path); + + // Exponential backoff: 3 attempts with delays 0ms, 100ms, 400ms + let backoff_delays = [0u64, 100, 400]; + let mut last_err = String::new(); + + for (attempt, delay_ms) in backoff_delays.iter().enumerate() { + if *delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(*delay_ms)).await; + } + + let mut req = self + .client + .post(&url) + .header("content-type", "application/octet-stream") + .body(payload.clone()); + + if let Some(ref secret) = self.cluster_secret { + req = req.header("x-cluster-secret", secret.as_str()); + } + + match req.send().await { + Ok(resp) => { + if resp.status().is_client_error() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(format!("Raft RPC {url} returned {status}: {body}")); + } + + if !resp.status().is_success() { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + last_err = format!("Raft RPC {url} returned {status}: {body}"); + tracing::debug!( + attempt = attempt + 1, + error = %last_err, + "Raft RPC failed, retrying" + ); + continue; + } + + let response_payload = resp + .bytes() + .await + .map_err(|e| format!("Read Raft RPC response from {url}: {e}"))? + .to_vec(); + + let response = match path { + "append-entries" => RaftMessage::AppendEntriesResponse { + payload: response_payload, + }, + "vote" => RaftMessage::VoteResponse { + payload: response_payload, + }, + "snapshot" => RaftMessage::InstallSnapshotResponse { + payload: response_payload, + }, + "snapshot-chunk" => { + // Could be SnapshotChunkAck or InstallSnapshotResponse + match serde_json::from_slice(&response_payload) { + Ok(msg) => msg, + Err(e) => { + tracing::warn!( + "Failed to deserialize snapshot-chunk response: {e}, \ + treating as InstallSnapshotResponse" + ); + RaftMessage::InstallSnapshotResponse { + payload: response_payload, + } + } + } + } + _ => unreachable!(), + }; + + return Ok(response); + } + Err(e) => { + last_err = format!("Raft RPC to {url}: {e}"); + tracing::debug!( + attempt = attempt + 1, + error = %last_err, + "Raft RPC failed, retrying" + ); + } + } + } + + Err(last_err) + }) + } +} + +/// Build a `rustls::ServerConfig` for the Raft RPC listener. +/// +/// If `cert_path` and `key_path` are provided, loads PEM files from disk. +/// Otherwise, generates a self-signed certificate using `rcgen` (TOFU model). +pub fn build_tls_server_config( + cert_path: Option<&str>, + key_path: Option<&str>, +) -> Result { + use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; + + let (cert_der, key_der): (CertificateDer<'static>, PrivateKeyDer<'static>) = + match (cert_path, key_path) { + (Some(cp), Some(kp)) => { + let cert_pem = std::fs::read(cp) + .map_err(|e| Error::Internal(format!("Read TLS cert {cp}: {e}")))?; + let key_pem = std::fs::read(kp) + .map_err(|e| Error::Internal(format!("Read TLS key {kp}: {e}")))?; + + let cert = rustls_pemfile::certs(&mut &cert_pem[..]) + .next() + .ok_or_else(|| Error::Internal("No certificate found in PEM file".into()))? + .map_err(|e| Error::Internal(format!("Parse TLS cert: {e}")))?; + + let key = rustls_pemfile::private_key(&mut &key_pem[..]) + .map_err(|e| Error::Internal(format!("Parse TLS key: {e}")))? + .ok_or_else(|| Error::Internal("No private key found in PEM file".into()))?; + + (cert, key) + } + _ => { + // Auto-generate self-signed cert (TOFU model, matching P2P transport) + let generated = rcgen::generate_simple_self_signed(vec![ + "localhost".to_string(), + "127.0.0.1".to_string(), + ]) + .map_err(|e| Error::Internal(format!("Generate self-signed cert: {e}")))?; + + let key = PrivatePkcs8KeyDer::from(generated.key_pair.serialize_der()); + let cert = CertificateDer::from(generated.cert); + (cert, key.into()) + } + }; + + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![cert_der], key_der) + .map_err(|e| Error::Internal(format!("TLS server config: {e}")))?; + + Ok(config) +} + +/// Initialize the Raft cluster on a `CortexServer`. +/// +/// This sets up the WAL, state machine, network factory, and Raft instance. +/// Must be called after `CortexServer::new()` and before `run()`. +/// +/// Returns the bind address used for the REST API (needed for join requests). +#[cfg(feature = "cluster")] +pub async fn init_cluster( + server: &mut CortexServer, + config: &ClusterConfig, + bind_addr: &str, + p2p_addr: &str, +) -> Result<(), Error> { + config.validate().map_err(|e| Error::Internal(e))?; + + let wal_dir = config.wal_dir.as_deref().unwrap_or("wal"); + let wal_path = std::path::Path::new(wal_dir); + + let log_store = match aingle_raft::log_store::CortexLogStore::open(wal_path) { + Ok(ls) => std::sync::Arc::new(ls), + Err(e) => return Err(Error::Internal(format!("Failed to initialize WAL: {e}"))), + }; + + server.state_mut().wal = Some(log_store.wal().clone()); + server.state_mut().cluster_secret = config.secret.clone(); + + let state_machine = std::sync::Arc::new( + aingle_raft::state_machine::CortexStateMachine::new( + server.state().graph.clone(), + server.state().memory.clone(), + ), + ); + + let resolver = std::sync::Arc::new(aingle_raft::network::NodeResolver::new()); + let node_id = config.node_id; + + resolver + .register( + node_id, + aingle_raft::CortexNode { + rest_addr: bind_addr.to_string(), + p2p_addr: p2p_addr.to_string(), + }, + ) + .await; + + let rpc_sender = std::sync::Arc::new(HttpRaftRpcSender::new( + config.secret.clone(), + config.tls, + )); + let network = aingle_raft::network::CortexNetworkFactory::new(resolver, rpc_sender); + + let raft_config = openraft::Config { + heartbeat_interval: 500, + election_timeout_min: 1500, + election_timeout_max: 3000, + ..Default::default() + }; + + let raft = openraft::Raft::new( + node_id, + std::sync::Arc::new(raft_config), + network, + log_store, + state_machine, + ) + .await + .map_err(|e| Error::Internal(format!("Failed to create Raft instance: {e}")))?; + + if config.peers.is_empty() { + // Bootstrap single-node cluster + let mut members = std::collections::BTreeMap::new(); + members.insert( + node_id, + aingle_raft::CortexNode { + rest_addr: bind_addr.to_string(), + p2p_addr: p2p_addr.to_string(), + }, + ); + if let Err(e) = raft.initialize(members).await { + use openraft::error::RaftError; + match e { + RaftError::APIError(openraft::error::InitializeError::NotAllowed(_)) => { + tracing::debug!("Raft already initialized: {e}"); + } + other => { + return Err(Error::Internal(format!( + "Raft initialization failed: {other}" + ))); + } + } + } + } else { + // Multi-node join with exponential backoff + let peers = config.peers.clone(); + let join_rest_addr = bind_addr.to_string(); + let join_p2p_addr = p2p_addr.to_string(); + let join_secret = config.secret.clone(); + let use_tls = config.tls; + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + let join_client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(5)) + .danger_accept_invalid_certs(use_tls) // TOFU for TLS + .build() + .unwrap(); + + let join_body = serde_json::json!({ + "node_id": node_id, + "rest_addr": join_rest_addr, + "p2p_addr": join_p2p_addr, + }); + + let scheme = if use_tls { "https" } else { "http" }; + let mut attempt = 0u32; + let max_attempts = 10; + loop { + attempt += 1; + let mut joined = false; + + for peer in &peers { + let url = format!("{scheme}://{peer}/api/v1/cluster/join"); + tracing::info!(url = %url, attempt, "Attempting to join cluster"); + + let mut req_builder = join_client.post(&url).json(&join_body); + + if let Some(ref secret) = join_secret { + req_builder = req_builder.header("x-cluster-secret", secret.as_str()); + } + + match req_builder.send().await { + Ok(resp) => { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + if status.is_success() { + tracing::info!( + peer = %peer, + response = %text, + "Successfully joined cluster" + ); + joined = true; + break; + } else { + tracing::warn!( + peer = %peer, + status = %status, + response = %text, + "Join request rejected, trying next peer" + ); + } + } + Err(e) => { + tracing::warn!( + peer = %peer, + error = %e, + "Failed to reach peer, trying next" + ); + } + } + } + + if joined { + break; + } + if attempt >= max_attempts { + tracing::error!("Exhausted {max_attempts} join attempts — giving up"); + break; + } + let base = std::time::Duration::from_secs(2u64.pow(attempt.min(5))); + let jitter = + std::time::Duration::from_millis(rand::random::() % 1000); + let backoff = base + jitter; + tracing::warn!(attempt, "Join failed, retrying in {:?}", backoff); + tokio::time::sleep(backoff).await; + } + }); + } + + // Set up TLS server config if cluster TLS is enabled + if config.tls { + let tls_config = build_tls_server_config( + config.tls_cert.as_deref(), + config.tls_key.as_deref(), + )?; + server.state_mut().tls_server_config = + Some(std::sync::Arc::new(tls_config)); + tracing::info!("Cluster TLS enabled for inter-node communication"); + } + + server.state_mut().raft = Some(raft); + server.state_mut().cluster_node_id = Some(node_id); + tracing::info!(node_id, "Raft consensus initialized"); + + Ok(()) +} diff --git a/crates/aingle_cortex/src/error.rs b/crates/aingle_cortex/src/error.rs index c0f95ed..0317164 100644 --- a/crates/aingle_cortex/src/error.rs +++ b/crates/aingle_cortex/src/error.rs @@ -97,6 +97,10 @@ pub enum Error { /// A conflict occurred, such as trying to create a resource that already exists. #[error("Conflict: {0}")] Conflict(String), + + /// The request should be redirected to another node (e.g., Raft leader). + #[error("Redirect to {0}")] + Redirect(String), } /// The standard JSON response body for an API error. @@ -136,6 +140,7 @@ impl Error { Error::Timeout(_) => StatusCode::REQUEST_TIMEOUT, Error::BadRequest(_) => StatusCode::BAD_REQUEST, Error::Conflict(_) => StatusCode::CONFLICT, + Error::Redirect(_) => StatusCode::TEMPORARY_REDIRECT, } } @@ -163,6 +168,7 @@ impl Error { Error::Timeout(_) => "TIMEOUT", Error::BadRequest(_) => "BAD_REQUEST", Error::Conflict(_) => "CONFLICT", + Error::Redirect(_) => "REDIRECT", } } } @@ -170,6 +176,17 @@ impl Error { impl IntoResponse for Error { fn into_response(self) -> Response { let status = self.status_code(); + + // For redirects, include a Location header so clients can follow + if let Error::Redirect(ref location) = self { + return ( + status, + [(axum::http::header::LOCATION, location.as_str())], + "Redirecting to leader", + ) + .into_response(); + } + let body = ErrorResponse { error: self.to_string(), code: self.error_code().to_string(), diff --git a/crates/aingle_cortex/src/lib.rs b/crates/aingle_cortex/src/lib.rs index faf2113..1875b7c 100644 --- a/crates/aingle_cortex/src/lib.rs +++ b/crates/aingle_cortex/src/lib.rs @@ -177,6 +177,8 @@ pub mod sparql; pub mod state; #[cfg(feature = "p2p")] pub mod p2p; +#[cfg(feature = "cluster")] +pub mod cluster_init; pub use client::{CortexClientConfig, CortexInternalClient}; pub use error::{Error, Result}; diff --git a/crates/aingle_cortex/src/main.rs b/crates/aingle_cortex/src/main.rs index a0875e2..faf3e48 100644 --- a/crates/aingle_cortex/src/main.rs +++ b/crates/aingle_cortex/src/main.rs @@ -91,10 +91,57 @@ async fn main() -> Result<(), Box> { } }; + // Parse and validate cluster config (feature-gated at compile time). + #[cfg(feature = "cluster")] + let cluster_config = { + let cfg = aingle_cortex::cluster_init::ClusterConfig::from_args(&args); + if cfg.enabled { + if let Err(e) = cfg.validate() { + eprintln!("Error: {e}"); + std::process::exit(1); + } + } + cfg + }; + + // Capture bind address before config is moved (used by cluster bootstrap) + #[allow(unused_variables)] + let bind_host = config.host.clone(); + #[allow(unused_variables)] + let bind_port = config.port; + // Create and run server #[allow(unused_mut)] let mut server = CortexServer::new(config)?; + // Initialize Raft cluster if enabled. + #[cfg(feature = "cluster")] + if cluster_config.enabled { + let this_rest_addr = format!("{}:{}", bind_host, bind_port); + #[cfg(feature = "p2p")] + let this_p2p_addr = format!("{}:{}", bind_host, p2p_config.port); + #[cfg(not(feature = "p2p"))] + let this_p2p_addr = "127.0.0.1:19091".to_string(); + + if let Err(e) = aingle_cortex::cluster_init::init_cluster( + &mut server, + &cluster_config, + &this_rest_addr, + &this_p2p_addr, + ) + .await + { + tracing::error!("Cluster initialization failed: {e}"); + std::process::exit(1); + } + + tracing::info!( + node_id = cluster_config.node_id, + peers = ?cluster_config.peers, + "Cluster mode enabled" + ); + } + // Keep a reference to the state for shutdown flush let state_for_shutdown = server.state().clone(); let snapshot_dir_for_shutdown = snapshot_dir.clone(); @@ -136,10 +183,26 @@ async fn main() -> Result<(), Box> { tokio::select! { _ = ctrl_c => { - tracing::info!("SIGINT received — flushing data..."); + tracing::info!("SIGINT received — shutting down..."); } _ = terminate => { - tracing::info!("SIGTERM received — flushing data..."); + tracing::info!("SIGTERM received — shutting down..."); + } + } + + // Gracefully shut down Raft before flushing data + #[cfg(feature = "cluster")] + if let Some(ref raft) = state_for_shutdown.raft { + tracing::info!("Shutting down Raft..."); + match tokio::time::timeout( + std::time::Duration::from_secs(10), + raft.shutdown(), + ) + .await + { + Ok(Ok(())) => tracing::info!("Raft shut down gracefully"), + Ok(Err(e)) => tracing::error!("Raft shutdown error: {e}"), + Err(_) => tracing::error!("Raft shutdown timed out after 10s"), } } @@ -181,6 +244,16 @@ fn print_help() { println!(" --p2p-peer Manual peer address (repeatable)"); println!(" --p2p-mdns Enable mDNS discovery"); println!(); + println!("CLUSTER OPTIONS (requires --features cluster):"); + println!(" --cluster Enable cluster mode (implies --p2p)"); + println!(" --cluster-node-id Unique node ID (u64, required)"); + println!(" --cluster-peers Comma-separated peer REST addresses"); + println!(" --cluster-wal-dir

WAL directory (default: wal/)"); + println!(" --cluster-secret Shared secret for internal RPC auth (min 16 bytes)"); + println!(" --cluster-tls Enable TLS for inter-node communication"); + println!(" --cluster-tls-cert TLS certificate PEM file"); + println!(" --cluster-tls-key TLS private key PEM file"); + println!(); println!("ENDPOINTS:"); println!(" REST API: http://:/api/v1/"); println!(" GraphQL: http://:/graphql"); diff --git a/crates/aingle_cortex/src/p2p/manager.rs b/crates/aingle_cortex/src/p2p/manager.rs index bed0b71..93bbf14 100644 --- a/crates/aingle_cortex/src/p2p/manager.rs +++ b/crates/aingle_cortex/src/p2p/manager.rs @@ -671,7 +671,15 @@ impl P2pManager { }) .await; } - _ => {} + other => { + // Raft RPC is routed over HTTP, not P2P QUIC. + // Log any unexpected messages instead of silently dropping. + tracing::debug!( + from = %addr, + msg_type = ?std::mem::discriminant(&other), + "Ignoring unexpected P2P message variant" + ); + } } } })); diff --git a/crates/aingle_cortex/src/p2p/message.rs b/crates/aingle_cortex/src/p2p/message.rs index 1c03e68..6d39f27 100644 --- a/crates/aingle_cortex/src/p2p/message.rs +++ b/crates/aingle_cortex/src/p2p/message.rs @@ -67,6 +67,39 @@ pub enum P2pMessage { TombstoneSync { tombstones: Vec, }, + // ── Raft / Cluster messages (feature: cluster) ────────────── + /// Raft AppendEntries RPC (serialized openraft request). + #[cfg(feature = "cluster")] + RaftAppendEntries { payload: Vec }, + /// Raft AppendEntries response. + #[cfg(feature = "cluster")] + RaftAppendEntriesResponse { payload: Vec }, + /// Raft Vote RPC. + #[cfg(feature = "cluster")] + RaftVote { payload: Vec }, + /// Raft Vote response. + #[cfg(feature = "cluster")] + RaftVoteResponse { payload: Vec }, + /// Raft InstallSnapshot RPC. + #[cfg(feature = "cluster")] + RaftInstallSnapshot { payload: Vec }, + /// Raft InstallSnapshot response. + #[cfg(feature = "cluster")] + RaftInstallSnapshotResponse { payload: Vec }, + /// Cluster membership join request. + #[cfg(feature = "cluster")] + ClusterJoin { + node_id: u64, + rest_addr: String, + p2p_addr: String, + }, + /// Cluster membership acknowledgement. + #[cfg(feature = "cluster")] + ClusterJoinAck { + accepted: bool, + leader_id: Option, + leader_addr: Option, + }, } /// Wire format for a tombstone marker. diff --git a/crates/aingle_cortex/src/rest/cluster.rs b/crates/aingle_cortex/src/rest/cluster.rs new file mode 100644 index 0000000..5408972 --- /dev/null +++ b/crates/aingle_cortex/src/rest/cluster.rs @@ -0,0 +1,413 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Cluster management REST endpoints. +//! +//! ## Endpoints +//! +//! - `GET /api/v1/cluster/status` — Node role, term, leader, members +//! - `POST /api/v1/cluster/join` — Request to join cluster +//! - `POST /api/v1/cluster/leave` — Graceful leave +//! - `GET /api/v1/cluster/members` — List members with replication lag +//! - `GET /api/v1/cluster/wal/stats` — WAL statistics +//! - `POST /api/v1/cluster/wal/verify` — Verify WAL hash chain integrity + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, + Json, +}; +use serde::{Deserialize, Serialize}; + +use crate::error::{Error, Result}; +use crate::rest::cluster_utils::validate_cluster_auth; +use crate::state::AppState; + +#[cfg(feature = "cluster")] +use openraft::type_config::async_runtime::watch::WatchReceiver; + +/// Cluster status response. +#[derive(Debug, Serialize)] +pub struct ClusterStatus { + pub node_id: u64, + pub role: String, + pub term: u64, + pub leader_id: Option, + pub leader_addr: Option, + pub members: Vec, + pub wal_last_seq: u64, + pub last_applied: u64, + pub commit_index: u64, +} + +/// Information about a single cluster member. +#[derive(Debug, Serialize)] +pub struct ClusterMember { + pub node_id: u64, + pub rest_addr: String, + pub p2p_addr: String, + pub role: String, + pub last_heartbeat: String, + pub replication_lag: u64, +} + +/// Request to join the cluster. +#[derive(Debug, Deserialize)] +pub struct JoinRequest { + pub node_id: u64, + pub rest_addr: String, + pub p2p_addr: String, +} + +/// Join response. +#[derive(Debug, Serialize)] +pub struct JoinResponse { + pub accepted: bool, + pub leader_id: Option, + pub leader_addr: Option, + pub message: String, +} + +/// WAL statistics response. +#[derive(Debug, Serialize)] +pub struct WalStatsResponse { + pub segment_count: usize, + pub total_size_bytes: u64, + pub last_seq: u64, + pub next_seq: u64, +} + +/// WAL verification response. +#[derive(Debug, Serialize)] +pub struct WalVerifyResponse { + pub valid: bool, + pub entries_checked: u64, + pub first_invalid_seq: Option, +} + +/// GET /api/v1/cluster/status +pub async fn cluster_status( + State(state): State, +) -> Result> { + let wal_last_seq = { + #[cfg(feature = "cluster")] + { + state.wal.as_ref().map(|w| w.last_seq()).unwrap_or(0) + } + #[cfg(not(feature = "cluster"))] + { 0u64 } + }; + + // Extract live Raft metrics when available + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let metrics = raft.metrics().borrow_watched().clone(); + + let role = format!("{:?}", metrics.state); + let term = metrics.current_term; + let leader_id = metrics.current_leader; + + let last_applied = metrics + .last_applied + .as_ref() + .map(|lid| lid.index) + .unwrap_or(0); + + let commit_index = metrics + .last_log_index + .unwrap_or(0); + + // Build member list from membership config + let membership = metrics.membership_config.membership(); + let members: Vec = membership + .nodes() + .map(|(nid, node)| ClusterMember { + node_id: *nid, + rest_addr: node.rest_addr.clone(), + p2p_addr: node.p2p_addr.clone(), + role: if Some(*nid) == leader_id { + "leader".to_string() + } else { + "follower".to_string() + }, + last_heartbeat: "N/A".to_string(), + replication_lag: 0, + }) + .collect(); + + // Resolve leader address from membership config (#13) + let leader_addr = leader_id.and_then(|lid| { + membership.nodes().find(|(nid, _)| **nid == lid).map(|(_, node)| node.rest_addr.clone()) + }); + + return Ok(Json(ClusterStatus { + node_id: state.cluster_node_id.unwrap_or(0), + role, + term, + leader_id, + leader_addr, + members, + wal_last_seq, + last_applied, + commit_index, + })); + } + + Ok(Json(ClusterStatus { + node_id: 0, + role: "standalone".to_string(), + term: 0, + leader_id: None, + leader_addr: None, + members: Vec::new(), + wal_last_seq, + last_applied: 0, + commit_index: 0, + })) +} + +/// POST /api/v1/cluster/join +pub async fn cluster_join( + State(state): State, + headers: HeaderMap, + Json(req): Json, +) -> Result<(StatusCode, Json)> { + validate_cluster_auth(&headers, &state)?; + + tracing::info!( + node_id = req.node_id, + rest_addr = %req.rest_addr, + "Cluster join request received" + ); + + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + // Check if this node is leader; if not, redirect (#14) + let metrics = raft.metrics().borrow_watched().clone(); + if metrics.current_leader != state.cluster_node_id { + let membership = metrics.membership_config.membership(); + let leader_addr = metrics.current_leader.and_then(|lid| { + membership.nodes().find(|(nid, _)| **nid == lid).map(|(_, node)| node.rest_addr.clone()) + }); + if let Some(ref addr) = leader_addr { + return Err(Error::Redirect(format!("http://{}/api/v1/cluster/join", addr))); + } + return Ok(( + StatusCode::CONFLICT, + Json(JoinResponse { + accepted: false, + leader_id: metrics.current_leader, + leader_addr, + message: "Not leader; leader unknown".to_string(), + }), + )); + } + + let node = aingle_raft::CortexNode { + rest_addr: req.rest_addr.clone(), + p2p_addr: req.p2p_addr.clone(), + }; + + // Add as learner first + match raft.add_learner(req.node_id, node, true).await { + Ok(_) => { + // Then promote to voter + let metrics = raft.metrics().borrow_watched().clone(); + let membership = metrics.membership_config.membership(); + let mut voter_ids: std::collections::BTreeSet = + membership.voter_ids().collect(); + voter_ids.insert(req.node_id); + // Resolve leader_addr for response + let leader_addr = metrics.current_leader.and_then(|lid| { + membership.nodes().find(|(nid, _)| **nid == lid).map(|(_, node)| node.rest_addr.clone()) + }); + match raft.change_membership(voter_ids.clone(), false).await { + Ok(_) => { + return Ok(( + StatusCode::OK, + Json(JoinResponse { + accepted: true, + leader_id: metrics.current_leader, + leader_addr, + message: format!("Node {} joined cluster", req.node_id), + }), + )); + } + Err(e) => { + // Rollback: remove orphaned learner + tracing::warn!( + "Membership change failed, removing learner {}", + req.node_id + ); + let mut rollback_ids = voter_ids; + rollback_ids.remove(&req.node_id); + let _ = raft.change_membership(rollback_ids, false).await; + return Ok(( + StatusCode::CONFLICT, + Json(JoinResponse { + accepted: false, + leader_id: metrics.current_leader, + leader_addr, + message: format!("Membership change failed: {e}"), + }), + )); + } + } + } + Err(e) => { + return Ok(( + StatusCode::CONFLICT, + Json(JoinResponse { + accepted: false, + leader_id: None, + leader_addr: None, + message: format!("Add learner failed: {e}"), + }), + )); + } + } + } + + Ok(( + StatusCode::OK, + Json(JoinResponse { + accepted: false, + leader_id: None, + leader_addr: None, + message: "Cluster mode not active on this node".to_string(), + }), + )) +} + +/// POST /api/v1/cluster/leave +pub async fn cluster_leave( + State(state): State, + headers: HeaderMap, +) -> Result { + validate_cluster_auth(&headers, &state)?; + tracing::info!("Cluster leave request received"); + + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + // Check if this node is leader; if not, redirect to leader (#14) + let metrics = raft.metrics().borrow_watched().clone(); + if metrics.current_leader != state.cluster_node_id { + let membership = metrics.membership_config.membership(); + let leader_addr = metrics.current_leader.and_then(|lid| { + membership.nodes().find(|(nid, _)| **nid == lid).map(|(_, node)| node.rest_addr.clone()) + }); + if let Some(ref addr) = leader_addr { + return Err(Error::Redirect(format!("http://{}/api/v1/cluster/leave", addr))); + } + return Err(Error::Internal("Not leader; leader unknown".to_string())); + } + + if let Some(node_id) = state.cluster_node_id { + let membership = metrics.membership_config.membership(); + let mut voter_ids: std::collections::BTreeSet = + membership.voter_ids().collect(); + voter_ids.remove(&node_id); + if !voter_ids.is_empty() { + if let Err(e) = raft.change_membership(voter_ids, false).await { + tracing::error!("Failed to leave cluster: {e}"); + return Err(Error::Internal(format!("Leave failed: {e}"))); + } + } + } + } + + Ok(StatusCode::OK) +} + +/// GET /api/v1/cluster/members +pub async fn cluster_members( + State(state): State, +) -> Result>> { + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let metrics = raft.metrics().borrow_watched().clone(); + let leader_id = metrics.current_leader; + + let membership = metrics.membership_config.membership(); + let members: Vec = membership + .nodes() + .map(|(nid, node)| ClusterMember { + node_id: *nid, + rest_addr: node.rest_addr.clone(), + p2p_addr: node.p2p_addr.clone(), + role: if Some(*nid) == leader_id { + "leader".to_string() + } else { + "follower".to_string() + }, + last_heartbeat: "N/A".to_string(), + replication_lag: 0, + }) + .collect(); + return Ok(Json(members)); + } + + Ok(Json(Vec::new())) +} + +/// GET /api/v1/cluster/wal/stats +pub async fn wal_stats( + State(state): State, +) -> Result> { + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + let stats = wal.stats().map_err(|e| Error::Internal(format!("WAL stats error: {e}")))?; + return Ok(Json(WalStatsResponse { + segment_count: stats.segment_count, + total_size_bytes: stats.total_size_bytes, + last_seq: stats.last_seq, + next_seq: stats.next_seq, + })); + } + + Ok(Json(WalStatsResponse { + segment_count: 0, + total_size_bytes: 0, + last_seq: 0, + next_seq: 0, + })) +} + +/// POST /api/v1/cluster/wal/verify +pub async fn wal_verify( + State(state): State, +) -> Result> { + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + let wal_dir = wal.dir(); + let reader = aingle_wal::WalReader::open(wal_dir) + .map_err(|e| Error::Internal(format!("WAL open failed: {e}")))?; + let result = reader + .verify_integrity() + .map_err(|e| Error::Internal(format!("WAL verify failed: {e}")))?; + return Ok(Json(WalVerifyResponse { + valid: result.valid, + entries_checked: result.entries_checked, + first_invalid_seq: result.first_invalid_seq, + })); + } + + Ok(Json(WalVerifyResponse { + valid: true, + entries_checked: 0, + first_invalid_seq: None, + })) +} + +/// Create the cluster sub-router. +pub fn cluster_router() -> axum::Router { + use axum::routing::{get, post}; + + axum::Router::new() + .route("/api/v1/cluster/status", get(cluster_status)) + .route("/api/v1/cluster/join", post(cluster_join)) + .route("/api/v1/cluster/leave", post(cluster_leave)) + .route("/api/v1/cluster/members", get(cluster_members)) + .route("/api/v1/cluster/wal/stats", get(wal_stats)) + .route("/api/v1/cluster/wal/verify", post(wal_verify)) +} diff --git a/crates/aingle_cortex/src/rest/cluster_utils.rs b/crates/aingle_cortex/src/rest/cluster_utils.rs new file mode 100644 index 0000000..42aa923 --- /dev/null +++ b/crates/aingle_cortex/src/rest/cluster_utils.rs @@ -0,0 +1,66 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Shared helpers for cluster-mode REST handlers. + +use axum::http::HeaderMap; +use crate::error::Error; +use crate::state::AppState; + +/// Convert a Raft `client_write` error into an appropriate HTTP error. +/// +/// If the error is `ForwardToLeader` with a known leader address, returns +/// `Error::Redirect` so the client gets a 307 with the leader's URL. +pub fn handle_raft_write_error( + e: openraft::error::RaftError< + aingle_raft::CortexTypeConfig, + openraft::error::ClientWriteError, + >, + _state: &AppState, +) -> Error { + use openraft::error::{ClientWriteError, RaftError}; + + match e { + RaftError::APIError(api_err) => match api_err { + ClientWriteError::ForwardToLeader(fwd) => { + if let Some(leader_node) = fwd.leader_node { + Error::Redirect(format!("http://{}", leader_node.rest_addr)) + } else { + Error::Internal("Not leader; leader unknown".to_string()) + } + } + ClientWriteError::ChangeMembershipError(e) => { + Error::Internal(format!("Membership change error: {e}")) + } + }, + RaftError::Fatal(f) => Error::Internal(format!("Raft fatal error: {f}")), + } +} + +/// Validate the `X-Cluster-Secret` header against the configured cluster secret. +/// +/// Returns `Ok(())` if the secret matches or if no secret is configured. +/// Returns `Err(Error::AuthError)` if the secret is missing or incorrect. +pub fn validate_cluster_auth(headers: &HeaderMap, state: &AppState) -> Result<(), Error> { + let expected = match &state.cluster_secret { + Some(s) if !s.is_empty() => s, + _ => return Ok(()), // No secret configured — allow all + }; + + let provided = headers + .get("x-cluster-secret") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + let expected_bytes = expected.as_bytes(); + let provided_bytes = provided.as_bytes(); + // Constant-time comparison to prevent timing side-channel attacks. + // Length check is not constant-time but doesn't leak the secret value. + if expected_bytes.len() != provided_bytes.len() + || subtle::ConstantTimeEq::ct_eq(expected_bytes, provided_bytes).unwrap_u8() != 1 + { + return Err(Error::AuthError("Invalid or missing cluster secret".into())); + } + + Ok(()) +} diff --git a/crates/aingle_cortex/src/rest/memory.rs b/crates/aingle_cortex/src/rest/memory.rs index 848d171..a59e92c 100644 --- a/crates/aingle_cortex/src/rest/memory.rs +++ b/crates/aingle_cortex/src/rest/memory.rs @@ -120,6 +120,52 @@ pub async fn remember( State(state): State, Json(req): Json, ) -> Result<(StatusCode, Json)> { + // Cluster mode: route through Raft + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let raft_req = aingle_raft::CortexRequest { + kind: aingle_wal::WalEntryKind::MemoryStore { + memory_id: String::new(), // assigned by state machine + entry_type: req.entry_type.clone(), + data: req.data.clone(), + importance: req.importance, + }, + }; + let resp = raft + .client_write(raft_req) + .await + .map_err(|e| handle_raft_write_error(e, &state))?; + + if !resp.response().success { + return Err(Error::Internal( + resp.response() + .detail + .clone() + .unwrap_or_else(|| "Raft memory store failed".to_string()), + )); + } + + let id = resp + .response() + .id + .clone() + .unwrap_or_else(|| "raft".to_string()); + + return Ok(( + StatusCode::CREATED, + Json(RememberResponse { id }), + )); + } + + // Guard: if Raft is initialized, all writes MUST go through Raft (#2). + #[cfg(feature = "cluster")] + if state.raft.is_some() { + return Err(Error::Internal("Raft initialized but write not routed through Raft".into())); + } + + // Non-cluster mode: direct write + #[cfg(feature = "cluster")] + let wal_data = req.data.clone(); let mut entry = MemoryEntry::new(&req.entry_type, req.data); if !req.tags.is_empty() { @@ -138,6 +184,17 @@ pub async fn remember( .remember(entry) .map_err(|e| Error::Internal(format!("Memory store failed: {e}")))?; + // Append to WAL (legacy cluster path) + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + wal.append(aingle_wal::WalEntryKind::MemoryStore { + memory_id: id.to_hex(), + entry_type: req.entry_type.clone(), + data: wal_data.clone(), + importance: req.importance, + }).map_err(|e| Error::Internal(format!("WAL append failed: {e}")))?; + } + Ok(( StatusCode::CREATED, Json(RememberResponse { @@ -183,11 +240,61 @@ pub async fn recall( pub async fn consolidate( State(state): State, ) -> Result> { + // Cluster mode: route through Raft so all nodes consolidate deterministically + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let raft_req = aingle_raft::CortexRequest { + kind: aingle_wal::WalEntryKind::MemoryConsolidate { + consolidated_count: 0, // state machine will compute real count + }, + }; + let resp = raft + .client_write(raft_req) + .await + .map_err(|e| handle_raft_write_error(e, &state))?; + + if !resp.response().success { + return Err(Error::Internal( + resp.response() + .detail + .clone() + .unwrap_or_else(|| "Raft consolidate failed".to_string()), + )); + } + + // The detail field contains the consolidated count from state machine + let count: usize = resp + .response() + .detail + .as_ref() + .and_then(|d| d.parse().ok()) + .unwrap_or(0); + + return Ok(Json(ConsolidateResponse { + consolidated: count, + })); + } + + // Guard: if Raft is initialized, all writes MUST go through Raft (#2). + #[cfg(feature = "cluster")] + if state.raft.is_some() { + return Err(Error::Internal("Raft initialized but write not routed through Raft".into())); + } + + // Non-cluster mode: direct consolidation let mut memory = state.memory.write().await; let count = memory .consolidate() .map_err(|e| Error::Internal(format!("Consolidation failed: {e}")))?; + // Append to WAL (legacy cluster path) + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + wal.append(aingle_wal::WalEntryKind::MemoryConsolidate { + consolidated_count: count, + }).map_err(|e| Error::Internal(format!("WAL append failed: {e}")))?; + } + Ok(Json(ConsolidateResponse { consolidated: count, })) @@ -212,6 +319,38 @@ pub async fn forget( State(state): State, Path(id): Path, ) -> Result { + // Cluster mode: route through Raft + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let raft_req = aingle_raft::CortexRequest { + kind: aingle_wal::WalEntryKind::MemoryForget { + memory_id: id.clone(), + }, + }; + let resp = raft + .client_write(raft_req) + .await + .map_err(|e| handle_raft_write_error(e, &state))?; + + if !resp.response().success { + return Err(Error::Internal( + resp.response() + .detail + .clone() + .unwrap_or_else(|| "Raft forget failed".to_string()), + )); + } + + return Ok(StatusCode::NO_CONTENT); + } + + // Guard: if Raft is initialized, all writes MUST go through Raft (#2). + #[cfg(feature = "cluster")] + if state.raft.is_some() { + return Err(Error::Internal("Raft initialized but write not routed through Raft".into())); + } + + // Non-cluster mode: direct delete let memory_id = MemoryId::from_hex(&id) .ok_or_else(|| Error::InvalidInput(format!("Invalid memory ID: {id}")))?; @@ -220,6 +359,14 @@ pub async fn forget( .forget(&memory_id) .map_err(|e| Error::NotFound(format!("Memory not found: {e}")))?; + // Append to WAL (legacy cluster path) + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + wal.append(aingle_wal::WalEntryKind::MemoryForget { + memory_id: id.clone(), + }).map_err(|e| Error::Internal(format!("WAL append failed: {e}")))?; + } + Ok(StatusCode::NO_CONTENT) } @@ -430,6 +577,10 @@ pub async fn rebuild_vector_index( // Helpers // ============================================================================ +/// Re-export shared Raft write error handler for this module. +#[cfg(feature = "cluster")] +use crate::rest::cluster_utils::handle_raft_write_error; + fn build_query(req: &RecallRequest) -> MemoryQuery { let mut query = if let Some(text) = &req.text { MemoryQuery::text(text) diff --git a/crates/aingle_cortex/src/rest/mod.rs b/crates/aingle_cortex/src/rest/mod.rs index bdbba03..315b445 100644 --- a/crates/aingle_cortex/src/rest/mod.rs +++ b/crates/aingle_cortex/src/rest/mod.rs @@ -30,6 +30,12 @@ //! - `POST /api/v1/assertions/verify-batch` - Batch verify assertion proofs pub mod audit; +#[cfg(feature = "cluster")] +pub mod cluster; +#[cfg(feature = "cluster")] +pub(crate) mod cluster_utils; +#[cfg(feature = "cluster")] +pub mod raft_rpc; mod memory; mod observability; #[cfg(feature = "p2p")] @@ -116,5 +122,11 @@ pub fn router() -> Router { #[cfg(feature = "p2p")] let router = router.merge(p2p::p2p_router()); + // Cluster endpoints (feature-gated) + #[cfg(feature = "cluster")] + let router = router + .merge(cluster::cluster_router()) + .merge(raft_rpc::raft_rpc_router()); + router } diff --git a/crates/aingle_cortex/src/rest/raft_rpc.rs b/crates/aingle_cortex/src/rest/raft_rpc.rs new file mode 100644 index 0000000..65d1c76 --- /dev/null +++ b/crates/aingle_cortex/src/rest/raft_rpc.rs @@ -0,0 +1,308 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Internal Raft RPC endpoints for inter-node communication. +//! +//! These endpoints handle Raft protocol messages (AppendEntries, Vote, +//! InstallSnapshot) over HTTP. They are used by `HttpRaftRpcSender` +//! on other nodes to drive the Raft consensus protocol. +//! +//! ## Endpoints +//! +//! - `POST /internal/raft/append-entries` — AppendEntries RPC +//! - `POST /internal/raft/vote` — Vote RPC +//! - `POST /internal/raft/snapshot` — Install full snapshot + +use axum::{ + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode}, + response::IntoResponse, +}; + +use crate::error::Error; +use crate::rest::cluster_utils::validate_cluster_auth; +use crate::state::AppState; + +type C = aingle_raft::CortexTypeConfig; + +/// POST /internal/raft/append-entries +/// +/// Receives a serialized `AppendEntriesRequest`, forwards to the local +/// Raft instance, and returns the serialized response. +pub async fn raft_append_entries( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + validate_cluster_auth(&headers, &state)?; + + let raft = state + .raft + .as_ref() + .ok_or_else(|| Error::Internal("Raft not initialized".into()))?; + + let req: openraft::raft::AppendEntriesRequest = serde_json::from_slice(&body) + .map_err(|e| Error::Internal(format!("Deserialize AppendEntries: {e}")))?; + + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + raft.append_entries(req), + ) + .await + .map_err(|_| Error::Timeout("AppendEntries RPC timed out (10s)".into()))? + .map_err(|e| Error::Internal(format!("AppendEntries failed: {e}")))?; + + let payload = serde_json::to_vec(&resp) + .map_err(|e| Error::Internal(format!("Serialize response: {e}")))?; + + Ok((StatusCode::OK, payload)) +} + +/// POST /internal/raft/vote +/// +/// Receives a serialized `VoteRequest`, forwards to the local +/// Raft instance, and returns the serialized response. +pub async fn raft_vote( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + validate_cluster_auth(&headers, &state)?; + + let raft = state + .raft + .as_ref() + .ok_or_else(|| Error::Internal("Raft not initialized".into()))?; + + let req: openraft::raft::VoteRequest = serde_json::from_slice(&body) + .map_err(|e| Error::Internal(format!("Deserialize Vote: {e}")))?; + + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + raft.vote(req), + ) + .await + .map_err(|_| Error::Timeout("Vote RPC timed out (10s)".into()))? + .map_err(|e| Error::Internal(format!("Vote failed: {e}")))?; + + let payload = serde_json::to_vec(&resp) + .map_err(|e| Error::Internal(format!("Serialize response: {e}")))?; + + Ok((StatusCode::OK, payload)) +} + +/// POST /internal/raft/snapshot +/// +/// Receives a serialized snapshot envelope (vote + meta + data), +/// forwards to the local Raft instance via `install_full_snapshot`. +pub async fn raft_snapshot( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + validate_cluster_auth(&headers, &state)?; + + let raft = state + .raft + .as_ref() + .ok_or_else(|| Error::Internal("Raft not initialized".into()))?; + + // The envelope matches what CortexNetworkConnection::full_snapshot serializes: + // { "vote": ..., "meta": ..., "data": [...] } + let envelope: serde_json::Value = serde_json::from_slice(&body) + .map_err(|e| Error::Internal(format!("Deserialize snapshot envelope: {e}")))?; + + let vote: openraft::type_config::alias::VoteOf = + serde_json::from_value(envelope["vote"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize vote: {e}")))?; + + let meta: openraft::type_config::alias::SnapshotMetaOf = + serde_json::from_value(envelope["meta"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize snapshot meta: {e}")))?; + + let data: Vec = serde_json::from_value(envelope["data"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize snapshot data: {e}")))?; + + let snapshot = openraft::Snapshot { + meta, + snapshot: std::io::Cursor::new(data), + }; + + let resp = tokio::time::timeout( + std::time::Duration::from_secs(60), + raft.install_full_snapshot(vote, snapshot), + ) + .await + .map_err(|_| Error::Timeout("InstallSnapshot RPC timed out (60s)".into()))? + .map_err(|e| Error::Internal(format!("InstallSnapshot failed: {e}")))?; + + let payload = serde_json::to_vec(&resp) + .map_err(|e| Error::Internal(format!("Serialize response: {e}")))?; + + Ok((StatusCode::OK, payload)) +} + +/// In-flight chunked snapshot buffer with creation timestamp for TTL. +struct SnapshotBuffer { + data: Vec, + expected_size: u64, + created_at: std::time::Instant, +} + +/// In-flight chunked snapshot buffers, keyed by snapshot_id. +/// Buffers older than `BUFFER_TTL` are evicted to prevent memory leaks +/// from abandoned transfers. +static SNAPSHOT_BUFFERS: std::sync::LazyLock< + dashmap::DashMap, +> = std::sync::LazyLock::new(dashmap::DashMap::new); + +/// Maximum time a partial snapshot buffer can live before eviction. +const BUFFER_TTL: std::time::Duration = std::time::Duration::from_secs(300); // 5 min + +/// Maximum total memory across all in-flight snapshot buffers (256 MB). +const MAX_BUFFER_MEMORY: usize = 256 * 1024 * 1024; + +/// Evict expired snapshot buffers to reclaim memory. +fn evict_stale_buffers() { + SNAPSHOT_BUFFERS.retain(|id, buf| { + let alive = buf.created_at.elapsed() < BUFFER_TTL; + if !alive { + tracing::warn!(snapshot_id = %id, "Evicting stale snapshot buffer"); + } + alive + }); +} + +/// POST /internal/raft/snapshot-chunk +/// +/// Receives a single chunk of a streamed snapshot. Chunks are buffered +/// in memory and assembled when the final chunk arrives. +pub async fn raft_snapshot_chunk( + State(state): State, + headers: HeaderMap, + body: Bytes, +) -> Result { + validate_cluster_auth(&headers, &state)?; + + // Evict stale buffers on each request (cheap: DashMap::retain is O(n)) + evict_stale_buffers(); + + let chunk: aingle_raft::network::RaftMessage = serde_json::from_slice(&body) + .map_err(|e| Error::Internal(format!("Deserialize snapshot chunk: {e}")))?; + + match chunk { + aingle_raft::network::RaftMessage::SnapshotChunk { + snapshot_id, + offset, + total_size, + is_final, + data, + } => { + // Reject snapshots that would exceed memory budget + if total_size as usize > MAX_BUFFER_MEMORY { + return Err(Error::Internal(format!( + "Snapshot too large: {total_size} bytes exceeds {MAX_BUFFER_MEMORY} limit" + ))); + } + + // Append chunk to buffer + let mut buf = SNAPSHOT_BUFFERS + .entry(snapshot_id.clone()) + .or_insert_with(|| SnapshotBuffer { + data: Vec::with_capacity(total_size as usize), + expected_size: total_size, + created_at: std::time::Instant::now(), + }); + + // Extend buffer to accommodate this chunk + let required = offset as usize + data.len(); + if buf.data.len() < required { + buf.data.resize(required, 0); + } + buf.data[offset as usize..offset as usize + data.len()].copy_from_slice(&data); + + if is_final { + // Remove buffer and validate completeness + let full_buf = SNAPSHOT_BUFFERS + .remove(&snapshot_id) + .ok_or_else(|| Error::Internal("Snapshot buffer missing on final chunk".into()))? + .1; + + if (full_buf.data.len() as u64) != full_buf.expected_size { + return Err(Error::Internal(format!( + "Snapshot size mismatch: got {} bytes, expected {}", + full_buf.data.len(), + full_buf.expected_size + ))); + } + + // Delegate to the monolithic snapshot handler + let result = install_full_snapshot_from_bytes(&state, &full_buf.data).await?; + Ok((StatusCode::OK, result)) + } else { + // ACK this chunk + let ack = aingle_raft::network::RaftMessage::SnapshotChunkAck { + snapshot_id, + next_offset: offset + data.len() as u64, + }; + let payload = serde_json::to_vec(&ack) + .map_err(|e| Error::Internal(format!("Serialize chunk ack: {e}")))?; + Ok((StatusCode::OK, payload)) + } + } + _ => Err(Error::Internal("Expected SnapshotChunk message".into())), + } +} + +/// Shared logic: install a full snapshot from its raw bytes. +async fn install_full_snapshot_from_bytes( + state: &AppState, + data: &[u8], +) -> Result, Error> { + let raft = state + .raft + .as_ref() + .ok_or_else(|| Error::Internal("Raft not initialized".into()))?; + + let envelope: serde_json::Value = serde_json::from_slice(data) + .map_err(|e| Error::Internal(format!("Deserialize snapshot envelope: {e}")))?; + + let vote: openraft::type_config::alias::VoteOf = + serde_json::from_value(envelope["vote"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize vote: {e}")))?; + + let meta: openraft::type_config::alias::SnapshotMetaOf = + serde_json::from_value(envelope["meta"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize snapshot meta: {e}")))?; + + let snap_data: Vec = serde_json::from_value(envelope["data"].clone()) + .map_err(|e| Error::Internal(format!("Deserialize snapshot data: {e}")))?; + + let snapshot = openraft::Snapshot { + meta, + snapshot: std::io::Cursor::new(snap_data), + }; + + let resp = tokio::time::timeout( + std::time::Duration::from_secs(60), + raft.install_full_snapshot(vote, snapshot), + ) + .await + .map_err(|_| Error::Timeout("InstallSnapshot timed out (60s)".into()))? + .map_err(|e| Error::Internal(format!("InstallSnapshot failed: {e}")))?; + + serde_json::to_vec(&resp) + .map_err(|e| Error::Internal(format!("Serialize response: {e}"))) +} + +/// Create the internal Raft RPC sub-router. +pub fn raft_rpc_router() -> axum::Router { + use axum::routing::post; + + axum::Router::new() + .route("/internal/raft/append-entries", post(raft_append_entries)) + .route("/internal/raft/vote", post(raft_vote)) + .route("/internal/raft/snapshot", post(raft_snapshot)) + .route("/internal/raft/snapshot-chunk", post(raft_snapshot_chunk)) +} diff --git a/crates/aingle_cortex/src/rest/triples.rs b/crates/aingle_cortex/src/rest/triples.rs index 2640926..fd928dd 100644 --- a/crates/aingle_cortex/src/rest/triples.rs +++ b/crates/aingle_cortex/src/rest/triples.rs @@ -16,6 +16,9 @@ use crate::rest::audit::AuditEntry; use crate::state::{AppState, Event}; use aingle_graph::{NodeId, Predicate, Triple, TripleId, TriplePattern, Value}; +#[cfg(feature = "cluster")] +use axum::http::HeaderMap; + /// Triple data transfer object #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TripleDto { @@ -150,6 +153,80 @@ pub async fn create_triple( let object: Value = req.object.clone().into(); + // Cluster mode: route writes through Raft + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let raft_req = aingle_raft::CortexRequest { + kind: aingle_wal::WalEntryKind::TripleInsert { + subject: req.subject.clone(), + predicate: req.predicate.clone(), + object: serde_json::to_value(&req.object).unwrap_or_default(), + triple_id: [0u8; 32], // State machine will compute the real ID + }, + }; + let resp = raft + .client_write(raft_req) + .await + .map_err(|e| handle_raft_write_error(e, &state))?; + + if !resp.response().success { + return Err(Error::Internal( + resp.response() + .detail + .clone() + .unwrap_or_else(|| "Raft apply failed".to_string()), + )); + } + + // State machine already applied the triple to GraphDB. + // Build response DTO from the request data, using the ID from the state machine. + let triple_id = resp.response().id.clone(); + let dto = TripleDto { + id: triple_id.clone(), + subject: req.subject.clone(), + predicate: req.predicate.clone(), + object: req.object.clone(), + created_at: Some(chrono::Utc::now().to_rfc3339()), + }; + + let hash = triple_id.unwrap_or_else(|| "raft".to_string()); + + // Record audit entry + { + let namespace = ns_ext + .as_ref() + .and_then(|axum::Extension(RequestNamespace(ns))| ns.clone()); + let mut audit = state.audit_log.write().await; + audit.record(AuditEntry { + timestamp: chrono::Utc::now().to_rfc3339(), + user_id: namespace.clone().unwrap_or_else(|| "anonymous".to_string()), + namespace, + action: "create".to_string(), + resource: format!("/api/v1/triples/{}", hash), + details: Some(format!("subject={}", req.subject)), + request_id: None, + }); + } + + // Broadcast event + state.broadcaster.broadcast(Event::TripleAdded { + hash, + subject: req.subject, + predicate: req.predicate, + object: serde_json::to_value(&req.object).unwrap_or_default(), + }); + + return Ok((StatusCode::CREATED, Json(dto))); + } + + // Guard: if Raft is initialized, all writes MUST go through Raft. + // Reaching here means Raft was skipped — prevent split-brain (#2). + #[cfg(feature = "cluster")] + if state.raft.is_some() { + return Err(Error::Internal("Raft initialized but write not routed through Raft".into())); + } + + // Non-cluster mode: direct write // Create the triple let triple = Triple::new( NodeId::named(&req.subject), @@ -163,6 +240,17 @@ pub async fn create_triple( graph.insert(triple.clone())? }; + // Append to WAL (cluster mode without Raft — legacy path) + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + wal.append(aingle_wal::WalEntryKind::TripleInsert { + subject: req.subject.clone(), + predicate: req.predicate.clone(), + object: serde_json::to_value(&req.object).unwrap_or_default(), + triple_id: *triple_id.as_bytes(), + }).map_err(|e| Error::Internal(format!("WAL append failed: {e}")))?; + } + // Record audit entry { let namespace = ns_ext @@ -191,13 +279,45 @@ pub async fn create_triple( Ok((StatusCode::CREATED, Json(triple.into()))) } +/// Parse X-Consistency header into a ConsistencyLevel. +#[cfg(feature = "cluster")] +fn parse_consistency_header(headers: &HeaderMap) -> aingle_raft::ConsistencyLevel { + headers + .get("x-consistency") + .and_then(|v| v.to_str().ok()) + .map(aingle_raft::ConsistencyLevel::from_header) + .unwrap_or_default() +} + /// Get a triple by hash /// /// GET /api/v1/triples/:id pub async fn get_triple( State(state): State, + #[cfg(feature = "cluster")] headers: HeaderMap, Path(id): Path, ) -> Result> { + // Apply consistency level for cluster reads + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let consistency = parse_consistency_header(&headers); + match consistency { + aingle_raft::ConsistencyLevel::Linearizable => { + raft.ensure_linearizable(openraft::raft::ReadPolicy::ReadIndex) + .await + .map_err(|e| Error::Internal(format!("Linearizable read: {e}")))?; + } + aingle_raft::ConsistencyLevel::Quorum => { + raft.ensure_linearizable(openraft::raft::ReadPolicy::LeaseRead) + .await + .map_err(|e| Error::Internal(format!("Quorum read: {e}")))?; + } + aingle_raft::ConsistencyLevel::Local => { + // Read from local state — no Raft check needed + } + } + } + let triple_id = TripleId::from_hex(&id) .ok_or_else(|| Error::InvalidInput(format!("Invalid triple ID: {}", id)))?; @@ -233,12 +353,55 @@ pub async fn delete_triple( } } + // Cluster mode: route deletes through Raft + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let raft_req = aingle_raft::CortexRequest { + kind: aingle_wal::WalEntryKind::TripleDelete { + triple_id: *triple_id.as_bytes(), + }, + }; + let resp = raft + .client_write(raft_req) + .await + .map_err(|e| handle_raft_write_error(e, &state))?; + + if !resp.response().success { + return Err(Error::Internal( + resp.response() + .detail + .clone() + .unwrap_or_else(|| "Raft delete failed".to_string()), + )); + } + + state + .broadcaster + .broadcast(Event::TripleDeleted { hash: id }); + return Ok(StatusCode::NO_CONTENT); + } + + // Guard: if Raft is initialized, all writes MUST go through Raft (#2). + #[cfg(feature = "cluster")] + if state.raft.is_some() { + return Err(Error::Internal("Raft initialized but write not routed through Raft".into())); + } + + // Non-cluster mode: direct delete let deleted = { let graph = state.graph.read().await; graph.delete(&triple_id)? }; if deleted { + // Append to WAL (legacy cluster path) + #[cfg(feature = "cluster")] + if let Some(ref wal) = state.wal { + wal.append(aingle_wal::WalEntryKind::TripleDelete { + triple_id: *triple_id.as_bytes(), + }).map_err(|e| Error::Internal(format!("WAL append failed: {e}")))?; + } + // Record audit entry { let namespace = ns_ext @@ -270,9 +433,29 @@ pub async fn delete_triple( /// GET /api/v1/triples pub async fn list_triples( State(state): State, + #[cfg(feature = "cluster")] headers: HeaderMap, ns_ext: Option>, Query(query): Query, ) -> Result> { + // Apply consistency level for cluster reads + #[cfg(feature = "cluster")] + if let Some(ref raft) = state.raft { + let consistency = parse_consistency_header(&headers); + match consistency { + aingle_raft::ConsistencyLevel::Linearizable => { + raft.ensure_linearizable(openraft::raft::ReadPolicy::ReadIndex) + .await + .map_err(|e| Error::Internal(format!("Consistent read: {e}")))?; + } + aingle_raft::ConsistencyLevel::Quorum => { + raft.ensure_linearizable(openraft::raft::ReadPolicy::LeaseRead) + .await + .map_err(|e| Error::Internal(format!("Consistent read: {e}")))?; + } + aingle_raft::ConsistencyLevel::Local => {} + } + } + let graph = state.graph.read().await; // Build pattern based on provided filters @@ -321,6 +504,10 @@ pub struct ListTriplesResponse { pub offset: usize, } +/// Re-export shared Raft write error handler for this module. +#[cfg(feature = "cluster")] +use crate::rest::cluster_utils::handle_raft_write_error; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/aingle_cortex/src/server.rs b/crates/aingle_cortex/src/server.rs index f4396c1..4fcfd23 100644 --- a/crates/aingle_cortex/src/server.rs +++ b/crates/aingle_cortex/src/server.rs @@ -213,6 +213,21 @@ impl CortexServer { let router = self.build_router(); + #[cfg(feature = "cluster")] + if let Some(ref tls_config) = self.state.tls_server_config { + info!("Starting Córtex API server on https://{}", addr); + + let tls_acceptor = + tokio_rustls::TlsAcceptor::from(tls_config.clone()); + let tcp_listener = tokio::net::TcpListener::bind(addr).await?; + let tls_listener = TlsListener { + inner: tcp_listener, + acceptor: tls_acceptor, + }; + axum::serve(tls_listener, router.into_make_service()).await?; + return Ok(()); + } + info!("Starting Córtex API server on http://{}", addr); info!("REST API: http://{}/api/v1", addr); #[cfg(feature = "graphql")] @@ -233,6 +248,7 @@ impl CortexServer { /// Runs the server with a graceful shutdown signal. /// /// The server will run until the `shutdown_signal` future completes. + /// If cluster TLS is configured, the server will accept HTTPS connections. pub async fn run_with_shutdown(self, shutdown_signal: F) -> Result<()> where F: std::future::Future + Send + 'static, @@ -243,6 +259,28 @@ impl CortexServer { let router = self.build_router(); + #[cfg(feature = "cluster")] + if let Some(ref tls_config) = self.state.tls_server_config { + info!("Starting Córtex API server on https://{}", addr); + + let tls_acceptor = + tokio_rustls::TlsAcceptor::from(tls_config.clone()); + let tcp_listener = tokio::net::TcpListener::bind(addr).await?; + let tls_listener = TlsListener { + inner: tcp_listener, + acceptor: tls_acceptor, + }; + axum::serve( + tls_listener, + router.into_make_service(), + ) + .with_graceful_shutdown(shutdown_signal) + .await?; + + info!("Córtex API server stopped"); + return Ok(()); + } + info!("Starting Córtex API server on http://{}", addr); let listener = tokio::net::TcpListener::bind(addr).await?; @@ -276,6 +314,48 @@ fn resolve_db_path(db_path: &Option) -> String { } } +// --------------------------------------------------------------------------- +// TLS Listener for cluster mode +// --------------------------------------------------------------------------- + +/// A TLS-wrapping listener that implements `axum::serve::Listener`. +/// +/// Accepts TCP connections, performs the TLS handshake, and yields +/// `TlsStream` to axum for request handling. Failed +/// handshakes are logged and retried automatically. +#[cfg(feature = "cluster")] +struct TlsListener { + inner: tokio::net::TcpListener, + acceptor: tokio_rustls::TlsAcceptor, +} + +#[cfg(feature = "cluster")] +impl axum::serve::Listener for TlsListener { + type Io = tokio_rustls::server::TlsStream; + type Addr = SocketAddr; + + async fn accept(&mut self) -> (Self::Io, Self::Addr) { + loop { + match self.inner.accept().await { + Ok((stream, addr)) => match self.acceptor.accept(stream).await { + Ok(tls_stream) => return (tls_stream, addr), + Err(e) => { + tracing::debug!("TLS handshake failed from {addr}: {e}"); + } + }, + Err(e) => { + tracing::debug!("TCP accept failed: {e}"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + } + } + + fn local_addr(&self) -> std::io::Result { + self.inner.local_addr() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/aingle_cortex/src/state.rs b/crates/aingle_cortex/src/state.rs index 0147abb..b6e36b2 100644 --- a/crates/aingle_cortex/src/state.rs +++ b/crates/aingle_cortex/src/state.rs @@ -43,6 +43,21 @@ pub struct AppState { /// P2P manager for multi-node triple synchronization. #[cfg(feature = "p2p")] pub p2p: Option>, + /// Write-Ahead Log for clustering. + #[cfg(feature = "cluster")] + pub wal: Option>, + /// Raft consensus instance for cluster coordination. + #[cfg(feature = "cluster")] + pub raft: Option>>, + /// This node's ID in the Raft cluster. + #[cfg(feature = "cluster")] + pub cluster_node_id: Option, + /// Shared secret for authenticating internal cluster RPCs. + #[cfg(feature = "cluster")] + pub cluster_secret: Option, + /// TLS server config for encrypting inter-node communication. + #[cfg(feature = "cluster")] + pub tls_server_config: Option>, } impl AppState { @@ -73,6 +88,16 @@ impl AppState { user_store, #[cfg(feature = "p2p")] p2p: None, + #[cfg(feature = "cluster")] + wal: None, + #[cfg(feature = "cluster")] + raft: None, + #[cfg(feature = "cluster")] + cluster_node_id: None, + #[cfg(feature = "cluster")] + cluster_secret: None, + #[cfg(feature = "cluster")] + tls_server_config: None, } } @@ -101,6 +126,16 @@ impl AppState { user_store, #[cfg(feature = "p2p")] p2p: None, + #[cfg(feature = "cluster")] + wal: None, + #[cfg(feature = "cluster")] + raft: None, + #[cfg(feature = "cluster")] + cluster_node_id: None, + #[cfg(feature = "cluster")] + cluster_secret: None, + #[cfg(feature = "cluster")] + tls_server_config: None, } } @@ -129,6 +164,16 @@ impl AppState { user_store, #[cfg(feature = "p2p")] p2p: None, + #[cfg(feature = "cluster")] + wal: None, + #[cfg(feature = "cluster")] + raft: None, + #[cfg(feature = "cluster")] + cluster_node_id: None, + #[cfg(feature = "cluster")] + cluster_secret: None, + #[cfg(feature = "cluster")] + tls_server_config: None, } } @@ -201,9 +246,20 @@ impl AppState { user_store, #[cfg(feature = "p2p")] p2p: None, + #[cfg(feature = "cluster")] + wal: None, + #[cfg(feature = "cluster")] + raft: None, + #[cfg(feature = "cluster")] + cluster_node_id: None, + #[cfg(feature = "cluster")] + cluster_secret: None, + #[cfg(feature = "cluster")] + tls_server_config: None, }) } + /// Flushes the graph database and saves the Ineru memory snapshot to disk. /// /// This should be called before shutdown or binary updates to ensure diff --git a/crates/aingle_cortex/tests/cluster_integration_test.rs b/crates/aingle_cortex/tests/cluster_integration_test.rs new file mode 100644 index 0000000..4969553 --- /dev/null +++ b/crates/aingle_cortex/tests/cluster_integration_test.rs @@ -0,0 +1,359 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Integration tests for 3-node Raft cluster. +//! +//! These tests boot multiple CortexServer instances with in-memory +//! databases, initialize Raft consensus, and verify write replication, +//! leader election, and graceful node leave. + +#![cfg(feature = "cluster")] + +use aingle_cortex::cluster_init::{ClusterConfig, init_cluster}; +use aingle_cortex::{CortexConfig, CortexServer}; +use std::time::Duration; +use tokio::time::sleep; + +/// Find a free TCP port by binding to port 0. +async fn free_port() -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener.local_addr().unwrap().port() +} + +/// Boots a single cluster node and returns (server_handle, shutdown_tx). +async fn boot_node( + node_id: u64, + port: u16, + peers: Vec, + secret: &str, + wal_dir: &str, +) -> (tokio::task::JoinHandle<()>, tokio::sync::watch::Sender) { + let mut config = CortexConfig::default() + .with_host("127.0.0.1") + .with_port(port); + config.db_path = Some(":memory:".to_string()); + + let mut server = CortexServer::new(config).unwrap(); + + let cluster_config = ClusterConfig { + enabled: true, + node_id, + peers, + wal_dir: Some(wal_dir.to_string()), + secret: Some(secret.to_string()), + tls: false, + tls_cert: None, + tls_key: None, + }; + + let bind_addr = format!("127.0.0.1:{port}"); + let p2p_port = free_port().await; + let p2p_addr = format!("127.0.0.1:{p2p_port}"); + + init_cluster(&mut server, &cluster_config, &bind_addr, &p2p_addr) + .await + .expect("cluster init failed"); + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false); + + let handle = tokio::spawn(async move { + let shutdown_signal = async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + }; + let _ = server.run_with_shutdown(shutdown_signal).await; + }); + + (handle, shutdown_tx) +} + +/// Gracefully shut down nodes with enough time for Raft to stop. +async fn shutdown_nodes(nodes: Vec<(tokio::task::JoinHandle<()>, tokio::sync::watch::Sender)>) { + for (_, tx) in &nodes { + tx.send(true).ok(); + } + for (h, _) in nodes { + // Raft shutdown can take up to 10s, give 15s total + let _ = tokio::time::timeout(Duration::from_secs(15), h).await; + } +} + +/// Wait for a node to report a leader via /api/v1/cluster/status. +async fn wait_for_leader(port: u16, timeout_secs: u64) -> Option { + let client = reqwest::Client::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout_secs); + + while tokio::time::Instant::now() < deadline { + if let Ok(resp) = client + .get(format!("http://127.0.0.1:{port}/api/v1/cluster/status")) + .send() + .await + { + if let Ok(body) = resp.json::().await { + if let Some(leader_id) = body["leader_id"].as_u64() { + if leader_id > 0 { + return Some(leader_id); + } + } + } + } + sleep(Duration::from_millis(250)).await; + } + None +} + +#[tokio::test] +async fn test_single_node_cluster_bootstrap() { + let tmp = tempfile::tempdir().unwrap(); + let port = free_port().await; + let wal_dir = tmp.path().join("wal1"); + + let (handle, shutdown_tx) = boot_node( + 1, + port, + vec![], // no peers = bootstrap + "test-secret-at-least-16-bytes", + wal_dir.to_str().unwrap(), + ) + .await; + + // Wait for server to start and Raft election to complete + sleep(Duration::from_secs(2)).await; + + // Should become leader of a single-node cluster + let leader = wait_for_leader(port, 10).await; + assert_eq!(leader, Some(1), "Single node should be leader"); + + // Write a triple + let client = reqwest::Client::new(); + let resp = client + .post(format!("http://127.0.0.1:{port}/api/v1/triples")) + .json(&serde_json::json!({ + "subject": "alice", + "predicate": "knows", + "object": "bob" + })) + .send() + .await + .unwrap(); + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + panic!("Write should succeed: {status} — {body}"); + } + + // Read it back + let resp = client + .get(format!( + "http://127.0.0.1:{port}/api/v1/triples?subject=alice" + )) + .send() + .await + .unwrap(); + assert!( + resp.status().is_success(), + "Read should succeed: {}", + resp.status() + ); + let body: serde_json::Value = resp.json().await.unwrap(); + let triples = body["triples"].as_array().expect("triples field should be an array"); + assert!( + !triples.is_empty(), + "Should find the triple we just wrote" + ); + + // Verify cluster members endpoint + let resp = client + .get(format!( + "http://127.0.0.1:{port}/api/v1/cluster/members" + )) + .send() + .await + .unwrap(); + assert!( + resp.status().is_success(), + "Members endpoint should succeed: {}", + resp.status() + ); + let members: Vec = resp.json().await.unwrap(); + assert_eq!(members.len(), 1, "Should have exactly 1 member"); + + // Shutdown + shutdown_nodes(vec![(handle, shutdown_tx)]).await; +} + +#[tokio::test] +async fn test_three_node_cluster_replication() { + let tmp = tempfile::tempdir().unwrap(); + let secret = "cluster-test-secret-32bytes!"; + + // Allocate 3 free ports + let port1 = free_port().await; + let port2 = free_port().await; + let port3 = free_port().await; + + let wal1 = tmp.path().join("wal1"); + let wal2 = tmp.path().join("wal2"); + let wal3 = tmp.path().join("wal3"); + + // Boot node 1 as bootstrap (no peers) + let (h1, tx1) = boot_node( + 1, + port1, + vec![], + secret, + wal1.to_str().unwrap(), + ) + .await; + + // Wait for Raft election (timeout_min=1500ms, give 2s + buffer) + sleep(Duration::from_secs(2)).await; + + // Wait for node 1 to become leader + let leader = wait_for_leader(port1, 10).await; + assert_eq!(leader, Some(1), "Node 1 should be leader after bootstrap"); + + // Boot node 2, joining via node 1 + let (h2, tx2) = boot_node( + 2, + port2, + vec![format!("127.0.0.1:{port1}")], + secret, + wal2.to_str().unwrap(), + ) + .await; + + // Boot node 3, joining via node 1 + let (h3, tx3) = boot_node( + 3, + port3, + vec![format!("127.0.0.1:{port1}")], + secret, + wal3.to_str().unwrap(), + ) + .await; + + // Wait for join operations to complete (join has 2s initial delay + processing) + sleep(Duration::from_secs(6)).await; + + // Verify members from the leader + let client = reqwest::Client::new(); + let resp = client + .get(format!( + "http://127.0.0.1:{port1}/api/v1/cluster/members" + )) + .send() + .await + .unwrap(); + let members: Vec = resp.json().await.unwrap(); + assert!( + members.len() >= 2, + "Should have at least 2 members (got {}): {:?}", + members.len(), + members + ); + + // Write a triple to the leader + let resp = client + .post(format!("http://127.0.0.1:{port1}/api/v1/triples")) + .json(&serde_json::json!({ + "subject": "cluster_test", + "predicate": "replicated_to", + "object": "all_nodes" + })) + .send() + .await + .unwrap(); + let status = resp.status(); + assert!( + status.is_success(), + "Write to leader should succeed: {status}" + ); + + // Wait for replication + sleep(Duration::from_secs(2)).await; + + // Read from follower node 2 — the triple should be replicated + let resp = client + .get(format!( + "http://127.0.0.1:{port2}/api/v1/triples?subject=cluster_test" + )) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "Read from node 2 failed: {}", resp.status()); + let body: serde_json::Value = resp.json().await.unwrap(); + let triples = body["triples"].as_array().expect("triples field should be an array"); + assert!( + !triples.is_empty(), + "Follower (node 2) should have the replicated triple" + ); + + // Also verify from follower node 3 + let resp = client + .get(format!( + "http://127.0.0.1:{port3}/api/v1/triples?subject=cluster_test" + )) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "Read from node 3 failed: {}", resp.status()); + let body: serde_json::Value = resp.json().await.unwrap(); + let triples = body["triples"].as_array().expect("triples field should be an array"); + assert!( + !triples.is_empty(), + "Follower (node 3) should have the replicated triple" + ); + + // Shutdown all nodes + shutdown_nodes(vec![(h1, tx1), (h2, tx2), (h3, tx3)]).await; +} + +#[tokio::test] +async fn test_cluster_wal_stats() { + let tmp = tempfile::tempdir().unwrap(); + let port = free_port().await; + let wal_dir = tmp.path().join("wal_stats"); + + let (handle, shutdown_tx) = boot_node( + 1, + port, + vec![], + "test-secret-at-least-16-bytes", + wal_dir.to_str().unwrap(), + ) + .await; + + sleep(Duration::from_secs(2)).await; + + let client = reqwest::Client::new(); + + // Check WAL stats endpoint + let resp = client + .get(format!( + "http://127.0.0.1:{port}/api/v1/cluster/wal/stats" + )) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "WAL stats failed: {}", resp.status()); + let stats: serde_json::Value = resp.json().await.unwrap(); + assert!(stats["segment_count"].is_number(), "segment_count should be a number: {stats}"); + + // Verify WAL integrity + let resp = client + .post(format!( + "http://127.0.0.1:{port}/api/v1/cluster/wal/verify" + )) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "WAL verify failed: {}", resp.status()); + let verify: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(verify["valid"], true, "WAL should be valid: {verify}"); + + shutdown_nodes(vec![(handle, shutdown_tx)]).await; +} diff --git a/crates/aingle_graph/Cargo.toml b/crates/aingle_graph/Cargo.toml index b4b6cdd..9d6e6f5 100644 --- a/crates/aingle_graph/Cargo.toml +++ b/crates/aingle_graph/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_graph" -version = "0.4.2" +version = "0.5.0" description = "Native GraphDB for AIngle - Semantic triple store with SPO indexes" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -20,8 +20,10 @@ rocksdb-backend = ["dep:rocksdb"] sqlite-backend = ["dep:rusqlite"] # RDF support rdf = ["dep:rio_turtle", "dep:rio_api"] +# CRDT conflict resolution (for clustering) +crdt = ["dep:uuid"] # Full features -full = ["sled-backend", "rocksdb-backend", "sqlite-backend", "rdf"] +full = ["sled-backend", "rocksdb-backend", "sqlite-backend", "rdf", "crdt"] [dependencies] # Serialization @@ -53,6 +55,9 @@ rusqlite = { version = "0.32", features = ["bundled"], optional = true } rio_turtle = { version = "0.8", optional = true } rio_api = { version = "0.8", optional = true } +# CRDT support (optional, for clustering) +uuid = { version = "1", features = ["v4", "serde"], optional = true } + [dev-dependencies] criterion = "0.5" tempfile = "3.26" diff --git a/crates/aingle_graph/src/crdt.rs b/crates/aingle_graph/src/crdt.rs new file mode 100644 index 0000000..e9e32e2 --- /dev/null +++ b/crates/aingle_graph/src/crdt.rs @@ -0,0 +1,297 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! CRDT conflict resolution for distributed triple stores. +//! +//! Provides Last-Writer-Wins (LWW) registers and Observed-Remove Sets +//! for deterministic conflict resolution when gossip-synced nodes have +//! concurrent writes. + +use crate::triple::{Triple, TripleId}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use uuid::Uuid; + +/// Last-Writer-Wins Register for triple conflicts. +/// +/// When two nodes write to the same triple ID concurrently, +/// the write with the latest timestamp wins. Ties are broken +/// deterministically by node ID (higher ID wins). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LwwTriple { + pub triple: Triple, + pub timestamp: DateTime, + pub node_id: u64, +} + +impl LwwTriple { + /// Create a new LWW-tagged triple. + pub fn new(triple: Triple, node_id: u64) -> Self { + Self { + triple, + timestamp: Utc::now(), + node_id, + } + } + + /// Create with an explicit timestamp. + pub fn with_timestamp(triple: Triple, timestamp: DateTime, node_id: u64) -> Self { + Self { + triple, + timestamp, + node_id, + } + } + + /// Merge two conflicting versions. Returns the winner. + pub fn merge(a: &LwwTriple, b: &LwwTriple) -> LwwTriple { + if a.timestamp > b.timestamp { + a.clone() + } else if b.timestamp > a.timestamp { + b.clone() + } else { + // Tie-break by node ID (deterministic: higher ID wins) + if a.node_id >= b.node_id { + a.clone() + } else { + b.clone() + } + } + } +} + +/// Observed-Remove Set for triple existence. +/// +/// Handles the case where one node inserts and another deletes +/// the same triple concurrently. Each insert generates a unique +/// tag; a remove only affects the tags that were observed at the +/// time of removal. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OrSet { + /// (triple_id bytes, add_tag) pairs — unique per insert operation. + adds: HashSet<([u8; 32], Uuid)>, + /// (triple_id bytes, add_tag) pairs that have been removed. + removes: HashSet<([u8; 32], Uuid)>, +} + +impl OrSet { + /// Create a new empty OR-Set. + pub fn new() -> Self { + Self { + adds: HashSet::new(), + removes: HashSet::new(), + } + } + + /// Insert a triple ID into the set, returning a unique tag. + pub fn insert(&mut self, id: &TripleId) -> Uuid { + let tag = Uuid::new_v4(); + self.adds.insert((*id.as_bytes(), tag)); + tag + } + + /// Remove all observed add-tags for this triple ID. + pub fn remove(&mut self, id: &TripleId) { + let id_bytes = *id.as_bytes(); + let to_remove: Vec<_> = self + .adds + .iter() + .filter(|(tid, _)| *tid == id_bytes) + .cloned() + .collect(); + for pair in to_remove { + self.adds.remove(&pair); + self.removes.insert(pair); + } + } + + /// Check if a triple ID is in the set (has at least one + /// non-removed add-tag). + pub fn contains(&self, id: &TripleId) -> bool { + let id_bytes = *id.as_bytes(); + self.adds + .iter() + .any(|(tid, tag)| *tid == id_bytes && !self.removes.contains(&(id_bytes, *tag))) + } + + /// Merge another OR-Set into this one. + /// + /// Union of adds and removes. Idempotent. + pub fn merge(&mut self, other: &OrSet) { + self.adds = self.adds.union(&other.adds).cloned().collect(); + self.removes = self.removes.union(&other.removes).cloned().collect(); + } + + /// Number of active (non-removed) entries. + pub fn len(&self) -> usize { + self.adds + .iter() + .filter(|pair| !self.removes.contains(pair)) + .count() + } + + /// Whether the set is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Default for OrSet { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{NodeId, Predicate, Value}; + use chrono::Duration; + + fn make_triple(subject: &str) -> Triple { + Triple::new( + NodeId::named(subject), + Predicate::named("knows"), + Value::String("bob".into()), + ) + } + + #[test] + fn test_lww_later_timestamp_wins() { + let triple = make_triple("alice"); + let now = Utc::now(); + let earlier = now - Duration::seconds(10); + + let a = LwwTriple::with_timestamp(triple.clone(), earlier, 1); + let b = LwwTriple::with_timestamp(triple, now, 2); + + let winner = LwwTriple::merge(&a, &b); + assert_eq!(winner.node_id, 2); // b wins (later timestamp) + } + + #[test] + fn test_lww_tiebreak_by_node_id() { + let triple = make_triple("alice"); + let same_time = Utc::now(); + + let a = LwwTriple::with_timestamp(triple.clone(), same_time, 1); + let b = LwwTriple::with_timestamp(triple, same_time, 2); + + let winner = LwwTriple::merge(&a, &b); + assert_eq!(winner.node_id, 2); // Higher node_id wins tie + } + + #[test] + fn test_lww_merge_commutative() { + let triple = make_triple("test"); + let now = Utc::now(); + + let a = LwwTriple::with_timestamp(triple.clone(), now, 1); + let b = LwwTriple::with_timestamp(triple, now - Duration::seconds(1), 2); + + let winner1 = LwwTriple::merge(&a, &b); + let winner2 = LwwTriple::merge(&b, &a); + assert_eq!(winner1.node_id, winner2.node_id); // Same result regardless of order + } + + #[test] + fn test_or_set_insert_contains() { + let mut set = OrSet::new(); + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + set.insert(&id); + assert!(set.contains(&id)); + assert_eq!(set.len(), 1); + } + + #[test] + fn test_or_set_remove() { + let mut set = OrSet::new(); + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + set.insert(&id); + assert!(set.contains(&id)); + + set.remove(&id); + assert!(!set.contains(&id)); + assert!(set.is_empty()); + } + + #[test] + fn test_or_set_concurrent_insert_remove() { + // Simulates: Node A inserts, Node B removes same ID (with tag from A), + // then Node A inserts again. The second insert should survive. + let mut set = OrSet::new(); + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + // First insert + set.insert(&id); + assert!(set.contains(&id)); + + // Remove (only removes tags observed so far) + set.remove(&id); + assert!(!set.contains(&id)); + + // Re-insert generates new tag + set.insert(&id); + assert!(set.contains(&id)); + } + + #[test] + fn test_or_set_merge() { + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + let mut set_a = OrSet::new(); + set_a.insert(&id); + + let mut set_b = OrSet::new(); + let triple2 = make_triple("bob"); + let id2 = TripleId::from_triple(&triple2); + set_b.insert(&id2); + + // Merge B into A + set_a.merge(&set_b); + assert!(set_a.contains(&id)); + assert!(set_a.contains(&id2)); + } + + #[test] + fn test_or_set_merge_idempotent() { + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + let mut set = OrSet::new(); + set.insert(&id); + + let snapshot = set.clone(); + set.merge(&snapshot); + + assert_eq!(set.len(), 1); // Merging with self doesn't duplicate + } + + #[test] + fn test_or_set_merge_with_removes() { + let triple = make_triple("alice"); + let id = TripleId::from_triple(&triple); + + let mut set_a = OrSet::new(); + set_a.insert(&id); + + // B gets A's state and removes the entry + let mut set_b = set_a.clone(); + set_b.remove(&id); + + // A doesn't know about the remove yet + assert!(set_a.contains(&id)); + assert!(!set_b.contains(&id)); + + // After merge, the remove propagates + set_a.merge(&set_b); + assert!(!set_a.contains(&id)); + } +} diff --git a/crates/aingle_graph/src/lib.rs b/crates/aingle_graph/src/lib.rs index 84c8e32..e806ebe 100644 --- a/crates/aingle_graph/src/lib.rs +++ b/crates/aingle_graph/src/lib.rs @@ -70,6 +70,8 @@ //! ``` pub mod backends; +#[cfg(feature = "crdt")] +pub mod crdt; pub mod error; pub mod index; pub mod node; diff --git a/crates/aingle_logic/Cargo.toml b/crates/aingle_logic/Cargo.toml index d125b16..8f67b8b 100644 --- a/crates/aingle_logic/Cargo.toml +++ b/crates/aingle_logic/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_logic" -version = "0.4.2" +version = "0.5.0" description = "Proof-of-Logic validation engine for AIngle semantic graphs" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -21,7 +21,7 @@ owl = [] [dependencies] # Graph database -aingle_graph = { version = "0.4", path = "../aingle_graph" } +aingle_graph = { version = "0.5", path = "../aingle_graph" } # Serialization serde = { version = "1.0", features = ["derive"] } diff --git a/crates/aingle_minimal/Cargo.toml b/crates/aingle_minimal/Cargo.toml index 4fd5e60..f4a4aa6 100644 --- a/crates/aingle_minimal/Cargo.toml +++ b/crates/aingle_minimal/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_minimal" -version = "0.4.2" +version = "0.5.0" description = "Ultra-light AIngle node for IoT devices (<1MB RAM)" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -124,10 +124,10 @@ embedded-hal = { version = "1.0", optional = true } embedded-hal-async = { version = "1.0", optional = true } # AI Memory (Ineru) -ineru = { version = "0.4", path = "../ineru", optional = true } +ineru = { version = "0.5", path = "../ineru", optional = true } # Kaneru (AI Agent Framework) -kaneru = { version = "0.4", path = "../kaneru", optional = true } +kaneru = { version = "0.5", path = "../kaneru", optional = true } # REST API server (lightweight HTTP) tiny_http = { version = "0.12", optional = true } diff --git a/crates/aingle_raft/Cargo.toml b/crates/aingle_raft/Cargo.toml new file mode 100644 index 0000000..476b964 --- /dev/null +++ b/crates/aingle_raft/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "aingle_raft" +version = "0.5.0" +description = "Raft consensus for AIngle clustering" +license = "Apache-2.0 OR LicenseRef-Commercial" +repository = "https://github.com/ApiliumCode/aingle" +homepage = "https://apilium.com" +documentation = "https://docs.rs/aingle_raft" +authors = ["Apilium Technologies "] +keywords = ["aingle", "raft", "consensus", "clustering"] +categories = ["database"] +edition = "2021" +rust-version = "1.83" + +[dependencies] +openraft = { version = "0.10.0-alpha.17", features = ["serde", "type-alias"] } +aingle_wal = { version = "0.5", path = "../aingle_wal" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["full"] } +bincode = "2" +blake3 = "1.8" +tracing = "0.1" +chrono = { version = "0.4", features = ["serde"] } +futures-util = "0.3" +anyerror = "0.1" +aingle_graph = { version = "0.5", path = "../aingle_graph", features = ["sled-backend"] } +ineru = { version = "0.5", path = "../ineru" } + +[dev-dependencies] +tempfile = "3.26" +tokio-test = "0.4" diff --git a/crates/aingle_raft/src/consistency.rs b/crates/aingle_raft/src/consistency.rs new file mode 100644 index 0000000..cee877b --- /dev/null +++ b/crates/aingle_raft/src/consistency.rs @@ -0,0 +1,57 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Consistency levels for read operations. + +use serde::{Deserialize, Serialize}; + +/// Configurable read consistency for cluster operations. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)] +pub enum ConsistencyLevel { + /// Read from local state (may be stale on followers). + #[default] + Local, + /// Read requires majority agreement. + Quorum, + /// Linearizable read (goes through Raft leader). + Linearizable, +} + +impl ConsistencyLevel { + /// Parse from a header string value. + pub fn from_header(value: &str) -> Self { + match value.to_lowercase().as_str() { + "quorum" => Self::Quorum, + "linearizable" => Self::Linearizable, + _ => Self::Local, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_consistency_default() { + assert_eq!(ConsistencyLevel::default(), ConsistencyLevel::Local); + } + + #[test] + fn test_from_header() { + assert_eq!(ConsistencyLevel::from_header("local"), ConsistencyLevel::Local); + assert_eq!(ConsistencyLevel::from_header("quorum"), ConsistencyLevel::Quorum); + assert_eq!(ConsistencyLevel::from_header("linearizable"), ConsistencyLevel::Linearizable); + assert_eq!(ConsistencyLevel::from_header("LOCAL"), ConsistencyLevel::Local); + assert_eq!(ConsistencyLevel::from_header("QUORUM"), ConsistencyLevel::Quorum); + assert_eq!(ConsistencyLevel::from_header("unknown"), ConsistencyLevel::Local); + } + + #[test] + fn test_serialization() { + let level = ConsistencyLevel::Quorum; + let json = serde_json::to_string(&level).unwrap(); + let back: ConsistencyLevel = serde_json::from_str(&json).unwrap(); + assert_eq!(back, ConsistencyLevel::Quorum); + } +} diff --git a/crates/aingle_raft/src/lib.rs b/crates/aingle_raft/src/lib.rs new file mode 100644 index 0000000..04c867c --- /dev/null +++ b/crates/aingle_raft/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Raft consensus for AIngle clustering. +//! +//! Uses openraft for leader election and log replication, +//! backed by the AIngle WAL for durable log storage. + +pub mod types; +pub mod log_store; +pub mod state_machine; +pub mod snapshot_builder; +pub mod network; +pub mod consistency; + +pub use types::{CortexTypeConfig, CortexRequest, CortexResponse, CortexNode, NodeId}; +pub use consistency::ConsistencyLevel; diff --git a/crates/aingle_raft/src/log_store.rs b/crates/aingle_raft/src/log_store.rs new file mode 100644 index 0000000..82bbc8a --- /dev/null +++ b/crates/aingle_raft/src/log_store.rs @@ -0,0 +1,667 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Raft log storage backed by WAL segments. +//! +//! Implements `RaftLogReader` and `RaftLogStorage` from openraft, +//! persisting entries as `WalEntryKind::RaftEntry` variants and +//! vote/committed state as JSON files alongside the WAL directory. + +use crate::types::CortexTypeConfig; +use aingle_wal::{WalEntryKind, WalWriter}; +use openraft::alias::{EntryOf, LogIdOf, VoteOf}; +use openraft::storage::{IOFlushed, LogState, RaftLogStorage}; +use openraft::RaftLogReader; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::io::{self, Write}; +use std::ops::RangeBounds; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; + +type C = CortexTypeConfig; +type Vote = VoteOf; +type LogId = LogIdOf; +type Entry = EntryOf; + +/// Durable Raft log store backed by the AIngle WAL. +/// +/// In-memory BTreeMap serves reads; WAL provides persistence. +/// Vote and committed state are persisted as JSON files. +pub struct CortexLogStore { + vote: RwLock>, + committed: RwLock>, + log: RwLock>, + purged_log_id: RwLock>, + /// Truncation boundary — entries with index > this are invalid. + truncated_after: RwLock>, + /// WAL writer for durable persistence. + wal: Arc, + /// Directory for vote/committed JSON files. + wal_dir: PathBuf, +} + +impl CortexLogStore { + /// Open or create a log store backed by the WAL at `wal_dir`. + /// + /// On recovery, reads WAL segments, filters `RaftEntry` variants, + /// rebuilds the in-memory BTreeMap, then applies persisted + /// truncation/purge boundaries to discard stale entries. + pub fn open(wal_dir: &Path) -> io::Result { + let wal = Arc::new(WalWriter::open(wal_dir)?); + + // Recover vote from disk + let vote = Self::load_vote(wal_dir)?; + + // Recover committed from disk + let committed = Self::load_committed(wal_dir)?; + + // Recover purged boundary from disk + let purged_log_id = Self::load_purged(wal_dir)?; + + // Recover truncation boundary from disk + let truncated_after = Self::load_truncated_after(wal_dir)?; + + // Rebuild log from WAL + let reader = aingle_wal::WalReader::open(wal_dir)?; + let wal_entries = reader.read_from(0)?; + let mut log = BTreeMap::new(); + + for wal_entry in &wal_entries { + if let WalEntryKind::RaftEntry { index, term: _, data } = &wal_entry.kind { + match serde_json::from_slice::(data) { + Ok(entry) => { + log.insert(*index, entry); + } + Err(e) => { + tracing::warn!( + index = index, + "Failed to deserialize RaftEntry from WAL: {}", + e + ); + } + } + } + } + + // Apply persisted boundaries: remove entries outside the valid range + if let Some(ref purged) = purged_log_id { + log.retain(|idx, _| *idx > purged.index); + } + if let Some(ref trunc) = truncated_after { + log.retain(|idx, _| *idx <= trunc.index); + } + + tracing::info!( + entries = log.len(), + vote = ?vote, + committed = ?committed, + purged = ?purged_log_id, + truncated_after = ?truncated_after, + "CortexLogStore recovered from WAL" + ); + + Ok(Self { + vote: RwLock::new(vote), + committed: RwLock::new(committed), + log: RwLock::new(log), + purged_log_id: RwLock::new(purged_log_id), + truncated_after: RwLock::new(truncated_after), + wal, + wal_dir: wal_dir.to_path_buf(), + }) + } + + /// Get the WAL writer reference. + pub fn wal(&self) -> &Arc { + &self.wal + } + + // --- Atomic file write --- + + /// Atomically write data to a file: write to .tmp, fsync, rename. + fn atomic_write(target: &Path, data: &[u8]) -> io::Result<()> { + let tmp = target.with_extension("tmp"); + { + let mut f = std::fs::File::create(&tmp)?; + f.write_all(data)?; + f.sync_all()?; + } + std::fs::rename(&tmp, target)?; + // fsync the parent directory to ensure the rename is durable + if let Some(parent) = target.parent() { + if let Ok(dir) = std::fs::File::open(parent) { + let _ = dir.sync_all(); + } + } + Ok(()) + } + + // --- Persistence helpers --- + + fn vote_path(dir: &Path) -> PathBuf { + dir.join("raft_vote.json") + } + + fn committed_path(dir: &Path) -> PathBuf { + dir.join("raft_committed.json") + } + + fn purged_path(dir: &Path) -> PathBuf { + dir.join("raft_purged.json") + } + + fn truncated_after_path(dir: &Path) -> PathBuf { + dir.join("raft_truncated_after.json") + } + + fn persist_vote(dir: &Path, vote: &Vote) -> io::Result<()> { + let data = serde_json::to_vec_pretty(vote) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Self::atomic_write(&Self::vote_path(dir), &data) + } + + fn load_vote(dir: &Path) -> io::Result> { + let path = Self::vote_path(dir); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read(&path)?; + let vote: Vote = serde_json::from_slice(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(Some(vote)) + } + + fn persist_committed(dir: &Path, committed: &Option) -> io::Result<()> { + let data = serde_json::to_vec_pretty(committed) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Self::atomic_write(&Self::committed_path(dir), &data) + } + + fn load_committed(dir: &Path) -> io::Result> { + let path = Self::committed_path(dir); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read(&path)?; + let committed: Option = serde_json::from_slice(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(committed) + } + + fn persist_purged(dir: &Path, purged: &LogId) -> io::Result<()> { + let data = serde_json::to_vec_pretty(purged) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Self::atomic_write(&Self::purged_path(dir), &data) + } + + fn load_purged(dir: &Path) -> io::Result> { + let path = Self::purged_path(dir); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read(&path)?; + let purged: LogId = serde_json::from_slice(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(Some(purged)) + } + + fn persist_truncated_after(dir: &Path, lid: &Option) -> io::Result<()> { + let data = serde_json::to_vec_pretty(lid) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Self::atomic_write(&Self::truncated_after_path(dir), &data) + } + + fn load_truncated_after(dir: &Path) -> io::Result> { + let path = Self::truncated_after_path(dir); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read(&path)?; + let lid: Option = serde_json::from_slice(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(lid) + } + + // --- Internal append (for IOFlushed callback pattern) --- + + async fn append_inner(&self, entries: I) -> Result<(), io::Error> + where + I: IntoIterator + Send, + I::IntoIter: Send, + { + // Collect all entries and serialize them first, then write ALL to + // WAL before touching the BTreeMap. This prevents a partial batch + // leaving the in-memory map inconsistent with WAL on failure (#11). + let batch: Vec<(u64, u64, Vec, Entry)> = entries + .into_iter() + .map(|entry| { + let index = entry.log_id.index; + let term = entry.log_id.leader_id.term; + let data = serde_json::to_vec(&entry) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok((index, term, data, entry)) + }) + .collect::, io::Error>>()?; + + // Write ALL to WAL first + for (index, term, ref data, _) in &batch { + self.wal + .append(WalEntryKind::RaftEntry { index: *index, term: *term, data: data.clone() }) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + } + + // Only update BTreeMap after all WAL writes succeed + let mut log = self.log.write().await; + for (index, _, _, entry) in batch { + log.insert(index, entry); + } + + Ok(()) + } + + // --- Legacy convenience methods --- + + pub async fn log_length(&self) -> u64 { + let log = self.log.read().await; + log.len() as u64 + } + + pub async fn last_log_id(&self) -> Option { + let log = self.log.read().await; + log.values().last().map(|e| e.log_id.clone()) + } +} + +// ============================================================================ +// RaftLogReader implementation +// ============================================================================ + +impl RaftLogReader for Arc { + async fn try_get_log_entries + Clone + Debug + Send>( + &mut self, + range: RB, + ) -> Result, io::Error> { + let log = self.log.read().await; + let entries: Vec = log.range(range).map(|(_, e)| e.clone()).collect(); + Ok(entries) + } + + async fn read_vote(&mut self) -> Result, io::Error> { + let v = self.vote.read().await; + Ok(v.clone()) + } +} + +// ============================================================================ +// RaftLogStorage implementation +// ============================================================================ + +impl RaftLogStorage for Arc { + type LogReader = Arc; + + async fn get_log_state(&mut self) -> Result, io::Error> { + // Hold both locks simultaneously to avoid TOCTOU race + let log = self.log.read().await; + let purged = self.purged_log_id.read().await; + + let last_log_id = log + .values() + .last() + .map(|e| e.log_id.clone()) + .or_else(|| purged.clone()); + + Ok(LogState { + last_purged_log_id: purged.clone(), + last_log_id, + }) + } + + async fn get_log_reader(&mut self) -> Self::LogReader { + Arc::clone(self) + } + + async fn save_vote(&mut self, vote: &Vote) -> Result<(), io::Error> { + CortexLogStore::persist_vote(&self.wal_dir, vote)?; + let mut v = self.vote.write().await; + *v = Some(vote.clone()); + Ok(()) + } + + async fn save_committed(&mut self, committed: Option) -> Result<(), io::Error> { + CortexLogStore::persist_committed(&self.wal_dir, &committed)?; + let mut c = self.committed.write().await; + *c = committed; + Ok(()) + } + + async fn read_committed(&mut self) -> Result, io::Error> { + let c = self.committed.read().await; + Ok(c.clone()) + } + + async fn append(&mut self, entries: I, callback: IOFlushed) -> Result<(), io::Error> + where + I: IntoIterator + Send, + I::IntoIter: Send, + { + // Always invoke the callback, even on error, to prevent openraft hangs. + let result = self.append_inner(entries).await; + callback.io_completed(result.as_ref().map(|_| ()).map_err(|e| { + io::Error::new(e.kind(), e.to_string()) + })); + result + } + + async fn truncate_after(&mut self, last_log_id: Option) -> Result<(), io::Error> { + let mut log = self.log.write().await; + + match last_log_id { + Some(ref lid) => { + let keys_to_remove: Vec = + log.range((lid.index + 1)..).map(|(k, _)| *k).collect(); + for k in keys_to_remove { + log.remove(&k); + } + } + None => { + log.clear(); + } + } + + // Persist truncation boundary so recovery filters out stale entries + let mut trunc = self.truncated_after.write().await; + *trunc = last_log_id.clone(); + CortexLogStore::persist_truncated_after(&self.wal_dir, &last_log_id)?; + + Ok(()) + } + + async fn purge(&mut self, log_id: LogId) -> Result<(), io::Error> { + let mut log = self.log.write().await; + + let keys_to_remove: Vec = log + .range(..=log_id.index) + .map(|(k, _)| *k) + .collect(); + for k in keys_to_remove { + log.remove(&k); + } + + // Persist purge boundary + let mut purged = self.purged_log_id.write().await; + *purged = Some(log_id.clone()); + CortexLogStore::persist_purged(&self.wal_dir, &log_id)?; + + // Clean up old WAL segments that are entirely below the purge point + let _ = self.wal.truncate_before(log_id.index); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openraft::entry::RaftEntry; + use openraft::vote::leader_id_adv::CommittedLeaderId; + use openraft::vote::RaftLeaderId; + + fn make_entry(index: u64, term: u64) -> Entry { + Entry::new_blank(openraft::LogId::new( + CommittedLeaderId::new(term, 0), + index, + )) + } + + #[tokio::test] + async fn test_log_store_open_empty() { + let dir = tempfile::tempdir().unwrap(); + let store = CortexLogStore::open(dir.path()).unwrap(); + let store = Arc::new(store); + + let mut reader = store.clone(); + assert!(reader.read_vote().await.unwrap().is_none()); + assert_eq!(store.log_length().await, 0); + } + + #[tokio::test] + async fn test_append_and_read() { + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![make_entry(1, 1), make_entry(2, 1), make_entry(3, 1)]; + + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..4).await.unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[0].log_id.index, 1); + assert_eq!(result[2].log_id.index, 3); + } + + #[tokio::test] + async fn test_vote_persistence() { + let dir = tempfile::tempdir().unwrap(); + + // Write vote + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + let vote = openraft::Vote::new(1, 0); + store_mut.save_vote(&vote).await.unwrap(); + } + + // Reopen and verify + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut reader = store.clone(); + let vote = reader.read_vote().await.unwrap(); + assert!(vote.is_some()); + assert_eq!(vote.unwrap().leader_id().term, 1); + } + } + + #[tokio::test] + async fn test_truncate_after() { + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![ + make_entry(1, 1), + make_entry(2, 1), + make_entry(3, 1), + make_entry(4, 1), + ]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + // Truncate after index 2 + let lid = openraft::LogId::new(CommittedLeaderId::new(1, 0), 2); + store_mut.truncate_after(Some(lid)).await.unwrap(); + + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..5).await.unwrap(); + assert_eq!(result.len(), 2); + } + + #[tokio::test] + async fn test_truncate_survives_restart() { + let dir = tempfile::tempdir().unwrap(); + + // Write entries and truncate + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![ + make_entry(1, 1), + make_entry(2, 1), + make_entry(3, 1), + make_entry(4, 1), + ]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + let lid = openraft::LogId::new(CommittedLeaderId::new(1, 0), 2); + store_mut.truncate_after(Some(lid)).await.unwrap(); + } + + // Reopen — truncated entries must NOT reappear + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..5).await.unwrap(); + assert_eq!(result.len(), 2, "truncated entries must not survive restart"); + } + } + + #[tokio::test] + async fn test_purge() { + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![ + make_entry(1, 1), + make_entry(2, 1), + make_entry(3, 1), + ]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + let purge_id = openraft::LogId::new(CommittedLeaderId::new(1, 0), 2); + store_mut.purge(purge_id).await.unwrap(); + + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..4).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].log_id.index, 3); + } + + #[tokio::test] + async fn test_purge_survives_restart() { + let dir = tempfile::tempdir().unwrap(); + + // Write entries and purge + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![ + make_entry(1, 1), + make_entry(2, 1), + make_entry(3, 1), + ]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + let purge_id = openraft::LogId::new(CommittedLeaderId::new(1, 0), 2); + store_mut.purge(purge_id).await.unwrap(); + } + + // Reopen — purged entries must NOT reappear + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..4).await.unwrap(); + assert_eq!(result.len(), 1, "purged entries must not survive restart"); + assert_eq!(result[0].log_id.index, 3); + + // purged_log_id should also be restored + let mut store_mut = store.clone(); + let state = store_mut.get_log_state().await.unwrap(); + assert!(state.last_purged_log_id.is_some()); + assert_eq!(state.last_purged_log_id.unwrap().index, 2); + } + } + + #[tokio::test] + async fn test_reopen_recovery() { + let dir = tempfile::tempdir().unwrap(); + + // Write entries + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![make_entry(1, 1), make_entry(2, 1)]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + } + + // Reopen and verify entries are recovered + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut reader = store.clone(); + let result = reader.try_get_log_entries(1..3).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].log_id.index, 1); + } + } + + #[tokio::test] + async fn test_committed_persistence() { + let dir = tempfile::tempdir().unwrap(); + + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + let lid = openraft::LogId::new(CommittedLeaderId::new(1, 0), 5); + store_mut.save_committed(Some(lid)).await.unwrap(); + } + + { + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + let committed = store_mut.read_committed().await.unwrap(); + assert!(committed.is_some()); + assert_eq!(committed.unwrap().index, 5); + } + } + + #[tokio::test] + async fn test_get_log_state() { + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new(CortexLogStore::open(dir.path()).unwrap()); + let mut store_mut = store.clone(); + + let entries = vec![make_entry(1, 1), make_entry(2, 1)]; + store_mut + .append(entries, IOFlushed::noop()) + .await + .unwrap(); + + let state = store_mut.get_log_state().await.unwrap(); + assert!(state.last_purged_log_id.is_none()); + assert_eq!(state.last_log_id.unwrap().index, 2); + } + + #[tokio::test] + async fn test_atomic_write_persists() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("test_atomic.json"); + CortexLogStore::atomic_write(&target, b"hello world").unwrap(); + let data = std::fs::read(&target).unwrap(); + assert_eq!(data, b"hello world"); + // tmp file should not exist + assert!(!target.with_extension("tmp").exists()); + } +} diff --git a/crates/aingle_raft/src/network.rs b/crates/aingle_raft/src/network.rs new file mode 100644 index 0000000..3dc2dfc --- /dev/null +++ b/crates/aingle_raft/src/network.rs @@ -0,0 +1,520 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Raft network layer — bridges openraft RPC to QUIC P2P transport. +//! +//! Implements `RaftNetworkFactory` and `RaftNetworkV2` to route Raft +//! protocol messages through the existing P2P transport. + +use crate::types::{CortexNode, CortexTypeConfig, NodeId}; +use anyerror::AnyError; +use openraft::error::{RPCError, ReplicationClosed, StreamingError, Unreachable}; +use openraft::network::{RPCOption, RaftNetworkFactory}; +use openraft::raft::{ + AppendEntriesRequest, AppendEntriesResponse, SnapshotResponse, VoteRequest, VoteResponse, +}; +use openraft::type_config::alias::{SnapshotOf, VoteOf}; +use openraft::RaftNetworkV2; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::future::Future; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::sync::RwLock; + +type C = CortexTypeConfig; + +// ============================================================================ +// Raft P2P message types +// ============================================================================ + +/// Raft-related P2P message types. +/// +/// These are serialized and sent over QUIC bidirectional streams. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum RaftMessage { + /// Raft AppendEntries RPC (serialized openraft request). + AppendEntries { payload: Vec }, + /// Raft AppendEntries response. + AppendEntriesResponse { payload: Vec }, + /// Raft Vote RPC. + Vote { payload: Vec }, + /// Raft Vote response. + VoteResponse { payload: Vec }, + /// Raft snapshot data (monolithic, for small snapshots). + InstallSnapshot { payload: Vec }, + /// Raft snapshot response. + InstallSnapshotResponse { payload: Vec }, + /// Snapshot chunk for streaming large snapshots. + SnapshotChunk { + snapshot_id: String, + offset: u64, + total_size: u64, + is_final: bool, + data: Vec, + }, + /// Acknowledgement for a snapshot chunk. + SnapshotChunkAck { + snapshot_id: String, + next_offset: u64, + }, + /// Cluster join request. + ClusterJoin { + node_id: u64, + rest_addr: String, + p2p_addr: String, + }, + /// Cluster join acknowledgement. + ClusterJoinAck { + accepted: bool, + leader_id: Option, + leader_addr: Option, + }, +} + +// ============================================================================ +// Node resolver +// ============================================================================ + +/// Node address resolver for the Raft network. +pub struct NodeResolver { + node_map: Arc>>, +} + +impl NodeResolver { + /// Create a new resolver with an initial set of nodes. + pub fn new() -> Self { + Self { + node_map: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Register a node. + pub async fn register(&self, node_id: NodeId, node: CortexNode) { + let mut map = self.node_map.write().await; + map.insert(node_id, node); + } + + /// Remove a node. + pub async fn unregister(&self, node_id: &NodeId) { + let mut map = self.node_map.write().await; + map.remove(node_id); + } + + /// Resolve a node ID to its address info. + pub async fn resolve(&self, node_id: &NodeId) -> Option { + let map = self.node_map.read().await; + map.get(node_id).cloned() + } + + /// Get all known nodes. + pub async fn all_nodes(&self) -> HashMap { + self.node_map.read().await.clone() + } + + /// Number of known nodes. + pub async fn node_count(&self) -> usize { + self.node_map.read().await.len() + } +} + +impl Default for NodeResolver { + fn default() -> Self { + Self::new() + } +} + +// ============================================================================ +// RPC sender abstraction +// ============================================================================ + +/// Trait for sending Raft RPC messages over the network. +/// +/// Implemented by the P2P transport to allow the Raft network layer +/// to send messages without depending on QUIC directly. +pub trait RaftRpcSender: Send + Sync + 'static { + fn send_rpc( + &self, + addr: SocketAddr, + msg: RaftMessage, + ) -> std::pin::Pin> + Send + '_>>; +} + +// ============================================================================ +// Network factory +// ============================================================================ + +/// Factory that creates per-target network connections for Raft RPC. +pub struct CortexNetworkFactory { + resolver: Arc, + rpc_sender: Arc, +} + +impl CortexNetworkFactory { + /// Create a new network factory. + pub fn new(resolver: Arc, rpc_sender: Arc) -> Self { + Self { + resolver, + rpc_sender, + } + } +} + +impl RaftNetworkFactory for CortexNetworkFactory { + type Network = CortexNetworkConnection; + + async fn new_client(&mut self, target: NodeId, node: &CortexNode) -> Self::Network { + // Use REST address for HTTP-based Raft RPC routing. + let addr: SocketAddr = node + .rest_addr + .parse() + .unwrap_or_else(|_| "127.0.0.1:8080".parse().unwrap()); + + CortexNetworkConnection { + target, + target_addr: addr, + rpc_sender: Arc::clone(&self.rpc_sender), + } + } +} + +// ============================================================================ +// Network connection (per-target) +// ============================================================================ + +/// A single Raft network connection to a target node. +pub struct CortexNetworkConnection { + target: NodeId, + target_addr: SocketAddr, + rpc_sender: Arc, +} + +impl RaftNetworkV2 for CortexNetworkConnection { + async fn append_entries( + &mut self, + rpc: AppendEntriesRequest, + option: RPCOption, + ) -> Result, RPCError> { + let payload = serde_json::to_vec(&rpc) + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + + let msg = RaftMessage::AppendEntries { payload }; + + let response = tokio::time::timeout( + option.hard_ttl(), + self.rpc_sender.send_rpc(self.target_addr, msg), + ) + .await + .map_err(|_| { + RPCError::Unreachable(Unreachable::new(&AnyError::error(format!( + "AppendEntries RPC to {} timed out after {:?}", + self.target_addr, + option.hard_ttl() + )))) + })? + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + + match response { + RaftMessage::AppendEntriesResponse { payload } => { + let resp: AppendEntriesResponse = serde_json::from_slice(&payload) + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + Ok(resp) + } + _ => Err(RPCError::Unreachable(Unreachable::new(&AnyError::error( + "unexpected response type for AppendEntries", + )))), + } + } + + async fn vote( + &mut self, + rpc: VoteRequest, + option: RPCOption, + ) -> Result, RPCError> { + let payload = serde_json::to_vec(&rpc) + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + + let msg = RaftMessage::Vote { payload }; + + let response = tokio::time::timeout( + option.hard_ttl(), + self.rpc_sender.send_rpc(self.target_addr, msg), + ) + .await + .map_err(|_| { + RPCError::Unreachable(Unreachable::new(&AnyError::error(format!( + "Vote RPC to {} timed out after {:?}", + self.target_addr, + option.hard_ttl() + )))) + })? + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + + match response { + RaftMessage::VoteResponse { payload } => { + let resp: VoteResponse = serde_json::from_slice(&payload) + .map_err(|e| RPCError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + Ok(resp) + } + _ => Err(RPCError::Unreachable(Unreachable::new(&AnyError::error( + "unexpected response type for Vote", + )))), + } + } + + async fn full_snapshot( + &mut self, + vote: VoteOf, + snapshot: SnapshotOf, + _cancel: impl Future + Send + 'static, + option: RPCOption, + ) -> Result, StreamingError> { + // Serialize full snapshot + metadata + let snap_data = serde_json::json!({ + "vote": vote, + "meta": snapshot.meta, + "data": snapshot.snapshot.into_inner(), + }); + let payload = serde_json::to_vec(&snap_data).map_err(|e| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(e))) + })?; + + // Use chunked transfer for payloads > 1MB to avoid timeouts + // and reduce memory pressure on the receiver. + const CHUNK_THRESHOLD: usize = 1024 * 1024; // 1MB + + if payload.len() > CHUNK_THRESHOLD { + return self + .send_chunked_snapshot(&payload, option) + .await; + } + + // Small snapshot: send monolithic + let msg = RaftMessage::InstallSnapshot { payload }; + + let response = tokio::time::timeout( + option.hard_ttl(), + self.rpc_sender.send_rpc(self.target_addr, msg), + ) + .await + .map_err(|_| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(format!( + "Snapshot RPC to {} timed out after {:?}", + self.target_addr, + option.hard_ttl() + )))) + })? + .map_err(|e| StreamingError::Unreachable(Unreachable::new(&AnyError::error(e))))?; + + match response { + RaftMessage::InstallSnapshotResponse { payload } => { + let resp: SnapshotResponse = serde_json::from_slice(&payload).map_err(|e| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(e))) + })?; + Ok(resp) + } + _ => Err(StreamingError::Unreachable(Unreachable::new( + &AnyError::error("unexpected response type for InstallSnapshot"), + ))), + } + } + +} + +impl CortexNetworkConnection { + /// Send a large snapshot in chunks, waiting for ACK after each chunk. + /// + /// Each chunk is sent sequentially with an ACK-per-chunk protocol. + /// The final chunk triggers snapshot installation on the receiver, + /// which returns an `InstallSnapshotResponse`. + async fn send_chunked_snapshot( + &self, + payload: &[u8], + option: RPCOption, + ) -> Result, StreamingError> { + const CHUNK_SIZE: usize = 512 * 1024; + let total_size = payload.len() as u64; + let snapshot_id = format!( + "snap-{}-{}", + self.target, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + ); + + let num_chunks = (payload.len() + CHUNK_SIZE - 1) / CHUNK_SIZE; + tracing::info!( + target_node = self.target, + total_bytes = total_size, + chunks = num_chunks, + "Streaming snapshot in chunks" + ); + + // Per-chunk timeout: use the caller's TTL divided by chunks (min 30s). + let per_chunk_timeout = std::cmp::max( + option.hard_ttl() / (num_chunks as u32 + 1), + std::time::Duration::from_secs(30), + ); + + let mut offset = 0u64; + while (offset as usize) < payload.len() { + let end = std::cmp::min(offset as usize + CHUNK_SIZE, payload.len()); + let chunk_data = payload[offset as usize..end].to_vec(); + let is_final = end == payload.len(); + + let msg = RaftMessage::SnapshotChunk { + snapshot_id: snapshot_id.clone(), + offset, + total_size, + is_final, + data: chunk_data, + }; + + let response = tokio::time::timeout( + per_chunk_timeout, + self.rpc_sender.send_rpc(self.target_addr, msg), + ) + .await + .map_err(|_| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(format!( + "Snapshot chunk at offset {offset} timed out after {per_chunk_timeout:?}" + )))) + })? + .map_err(|e| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(e))) + })?; + + match response { + // Final chunk returns the install response + RaftMessage::InstallSnapshotResponse { payload } => { + let resp: SnapshotResponse = + serde_json::from_slice(&payload).map_err(|e| { + StreamingError::Unreachable(Unreachable::new(&AnyError::error(e))) + })?; + return Ok(resp); + } + // Intermediate ACK — advance offset + RaftMessage::SnapshotChunkAck { next_offset, .. } if !is_final => { + offset = next_offset; + } + // Got ACK on what should have been the final chunk — receiver + // didn't install yet (shouldn't happen, but handle gracefully) + RaftMessage::SnapshotChunkAck { .. } => { + return Err(StreamingError::Unreachable(Unreachable::new( + &AnyError::error( + "received SnapshotChunkAck for final chunk instead of InstallSnapshotResponse" + ), + ))); + } + other => { + return Err(StreamingError::Unreachable(Unreachable::new( + &AnyError::error(format!( + "unexpected response for snapshot chunk: {:?}", + std::mem::discriminant(&other) + )), + ))); + } + } + } + + Err(StreamingError::Unreachable(Unreachable::new( + &AnyError::error("snapshot transfer ended without a final response"), + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_raft_message_serialization() { + let msg = RaftMessage::AppendEntries { + payload: vec![1, 2, 3], + }; + let json = serde_json::to_string(&msg).unwrap(); + let back: RaftMessage = serde_json::from_str(&json).unwrap(); + assert!(matches!(back, RaftMessage::AppendEntries { .. })); + } + + #[test] + fn test_cluster_join_roundtrip() { + let msg = RaftMessage::ClusterJoin { + node_id: 42, + rest_addr: "127.0.0.1:8080".into(), + p2p_addr: "127.0.0.1:19091".into(), + }; + let json = serde_json::to_string(&msg).unwrap(); + let back: RaftMessage = serde_json::from_str(&json).unwrap(); + match back { + RaftMessage::ClusterJoin { + node_id, + rest_addr, + p2p_addr, + } => { + assert_eq!(node_id, 42); + assert_eq!(rest_addr, "127.0.0.1:8080"); + assert_eq!(p2p_addr, "127.0.0.1:19091"); + } + _ => panic!("wrong variant"), + } + } + + #[test] + fn test_cluster_join_ack() { + let msg = RaftMessage::ClusterJoinAck { + accepted: true, + leader_id: Some(1), + leader_addr: Some("127.0.0.1:8080".into()), + }; + let json = serde_json::to_string(&msg).unwrap(); + let back: RaftMessage = serde_json::from_str(&json).unwrap(); + match back { + RaftMessage::ClusterJoinAck { + accepted, + leader_id, + .. + } => { + assert!(accepted); + assert_eq!(leader_id, Some(1)); + } + _ => panic!("wrong variant"), + } + } + + #[tokio::test] + async fn test_node_resolver() { + let resolver = NodeResolver::new(); + + resolver + .register( + 1, + CortexNode { + rest_addr: "127.0.0.1:8080".into(), + p2p_addr: "127.0.0.1:19091".into(), + }, + ) + .await; + + resolver + .register( + 2, + CortexNode { + rest_addr: "127.0.0.1:8081".into(), + p2p_addr: "127.0.0.1:19092".into(), + }, + ) + .await; + + assert_eq!(resolver.node_count().await, 2); + + let node = resolver.resolve(&1).await; + assert!(node.is_some()); + assert_eq!(node.unwrap().rest_addr, "127.0.0.1:8080"); + + resolver.unregister(&1).await; + assert_eq!(resolver.node_count().await, 1); + assert!(resolver.resolve(&1).await.is_none()); + } +} diff --git a/crates/aingle_raft/src/snapshot_builder.rs b/crates/aingle_raft/src/snapshot_builder.rs new file mode 100644 index 0000000..5fbb0b2 --- /dev/null +++ b/crates/aingle_raft/src/snapshot_builder.rs @@ -0,0 +1,140 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Snapshot builder for the Raft state machine. + +use crate::state_machine::{ClusterSnapshot, TripleSnapshot}; +use crate::types::CortexTypeConfig; +use aingle_graph::GraphDB; +use ineru::IneruMemory; +use openraft::alias::LogIdOf; +use openraft::storage::{RaftSnapshotBuilder, Snapshot, SnapshotMeta}; +use openraft::type_config::alias::{SnapshotOf, StoredMembershipOf}; +use std::io; +use std::io::Cursor; +use std::sync::Arc; +use tokio::sync::RwLock; + +type C = CortexTypeConfig; +type LogId = LogIdOf; + +/// Builds a point-in-time snapshot of the graph + memory state. +pub struct CortexSnapshotBuilder { + pub graph: Arc>, + pub memory: Arc>, + pub last_applied: Option, + pub last_membership: StoredMembershipOf, +} + +impl RaftSnapshotBuilder for CortexSnapshotBuilder { + async fn build_snapshot(&mut self) -> Result, io::Error> { + // Acquire both locks simultaneously for an atomic snapshot + let graph = self.graph.read().await; + let memory = self.memory.read().await; + + let triples = { + let all = graph + .find(aingle_graph::TriplePattern::any()) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + + all.into_iter() + .map(|t| TripleSnapshot { + subject: t.subject.to_string(), + predicate: t.predicate.to_string(), + object: value_to_json(&t.object), + }) + .collect::>() + }; + + let ineru_ltm = memory + .export_snapshot() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + + // Drop locks before serialization to reduce hold time + drop(graph); + drop(memory); + + let (last_applied_index, last_applied_term) = match &self.last_applied { + Some(lid) => (lid.index, lid.leader_id.term), + None => (0, 0), + }; + + let snapshot = ClusterSnapshot { + triples, + ineru_ltm, + last_applied_index, + last_applied_term, + checksum: String::new(), + }; + + let data = snapshot + .to_bytes() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + let snapshot_id = format!( + "snap-{}-{}", + last_applied_term, last_applied_index + ); + + let meta = SnapshotMeta { + last_log_id: self.last_applied.clone(), + last_membership: self.last_membership.clone(), + snapshot_id, + }; + + Ok(Snapshot { + meta, + snapshot: Cursor::new(data), + }) + } +} + +fn value_to_json(v: &aingle_graph::Value) -> serde_json::Value { + match v { + aingle_graph::Value::String(s) => serde_json::Value::String(s.clone()), + aingle_graph::Value::Integer(i) => serde_json::json!(*i), + aingle_graph::Value::Float(f) => serde_json::json!(*f), + aingle_graph::Value::Boolean(b) => serde_json::json!(*b), + aingle_graph::Value::Json(j) => j.clone(), + aingle_graph::Value::Node(n) => serde_json::json!({ "node": n.to_string() }), + aingle_graph::Value::DateTime(dt) => serde_json::Value::String(dt.clone()), + aingle_graph::Value::Null => serde_json::Value::Null, + _ => serde_json::Value::String(format!("{:?}", v)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_build_snapshot() { + use openraft::vote::leader_id_adv::CommittedLeaderId; + use openraft::vote::RaftLeaderId; + + let graph = GraphDB::memory().unwrap(); + // Insert test data + let triple = aingle_graph::Triple::new( + aingle_graph::NodeId::named("alice"), + aingle_graph::Predicate::named("knows"), + aingle_graph::Value::String("bob".into()), + ); + graph.insert(triple).unwrap(); + + let memory = IneruMemory::agent_mode(); + + let mut builder = CortexSnapshotBuilder { + graph: Arc::new(RwLock::new(graph)), + memory: Arc::new(RwLock::new(memory)), + last_applied: Some(openraft::LogId::new( + CommittedLeaderId::new(1, 0), + 5, + )), + last_membership: openraft::StoredMembership::default(), + }; + + let snap = builder.build_snapshot().await.unwrap(); + assert_eq!(snap.meta.last_log_id.as_ref().unwrap().index, 5); + assert!(!snap.snapshot.into_inner().is_empty()); + } +} diff --git a/crates/aingle_raft/src/state_machine.rs b/crates/aingle_raft/src/state_machine.rs new file mode 100644 index 0000000..75b1bab --- /dev/null +++ b/crates/aingle_raft/src/state_machine.rs @@ -0,0 +1,832 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Raft state machine — applies committed entries to GraphDB + Ineru. + +use crate::snapshot_builder::CortexSnapshotBuilder; +use crate::types::{CortexResponse, CortexTypeConfig}; +use aingle_graph::GraphDB; +use aingle_wal::WalEntryKind; +use futures_util::StreamExt; +use ineru::IneruMemory; +use openraft::alias::LogIdOf; +use openraft::entry::RaftPayload; +use openraft::storage::{EntryResponder, RaftStateMachine, Snapshot}; +use openraft::type_config::alias::{SnapshotMetaOf, SnapshotOf, StoredMembershipOf}; +use openraft::StoredMembership; +use serde::{Deserialize, Serialize}; +use std::io; +use std::io::Cursor; +use std::sync::Arc; +use tokio::sync::RwLock; + +type C = CortexTypeConfig; +type LogId = LogIdOf; + +/// Raft state machine that applies committed mutations to GraphDB + Ineru. +/// +/// When Raft commits an entry, the state machine applies it +/// to the local graph database and memory system. +pub struct CortexStateMachine { + graph: Arc>, + memory: Arc>, + last_applied: RwLock>, + last_membership: RwLock>, + current_snapshot: RwLock, Vec)>>, + /// Count of applied mutations (for metrics). + applied_count: RwLock, +} + +impl CortexStateMachine { + /// Create a new state machine connected to shared GraphDB and IneruMemory. + pub fn new(graph: Arc>, memory: Arc>) -> Self { + Self { + graph, + memory, + last_applied: RwLock::new(None), + last_membership: RwLock::new(StoredMembership::default()), + current_snapshot: RwLock::new(None), + applied_count: RwLock::new(0), + } + } + + /// Apply a mutation from the WAL entry kind to the real stores. + pub async fn apply_mutation(&self, kind: &WalEntryKind) -> CortexResponse { + let mut count = self.applied_count.write().await; + *count += 1; + + match kind { + WalEntryKind::TripleInsert { + subject, + predicate, + object, + triple_id: _, + } => { + let value = json_to_value(object); + let triple = aingle_graph::Triple::new( + aingle_graph::NodeId::named(subject), + aingle_graph::Predicate::named(predicate), + value, + ); + let graph = self.graph.read().await; + match graph.insert(triple) { + Ok(id) => { + tracing::debug!(subject, predicate, "Applied TripleInsert"); + CortexResponse { + success: true, + detail: None, + id: Some(id.to_hex()), + } + } + Err(e) => { + tracing::error!("TripleInsert failed (potential state divergence): {e}"); + CortexResponse { + success: false, + detail: Some(format!("Insert failed: {e}")), + id: None, + } + } + } + } + WalEntryKind::TripleDelete { triple_id } => { + let tid = aingle_graph::TripleId::new(*triple_id); + let graph = self.graph.read().await; + match graph.delete(&tid) { + Ok(_) => { + tracing::debug!("Applied TripleDelete"); + CortexResponse { + success: true, + detail: None, + id: None, + } + } + Err(e) => { + tracing::error!("TripleDelete failed (potential state divergence): {e}"); + CortexResponse { + success: false, + detail: Some(format!("Delete failed: {e}")), + id: None, + } + } + } + } + WalEntryKind::MemoryStore { + memory_id: _, + entry_type, + data, + importance, + } => { + let entry = + ineru::MemoryEntry::new(entry_type, data.clone()).with_importance(*importance); + let mut memory = self.memory.write().await; + match memory.remember(entry) { + Ok(id) => CortexResponse { + success: true, + detail: None, + id: Some(id.to_hex()), + }, + Err(e) => CortexResponse { + success: false, + detail: Some(format!("MemoryStore failed: {e}")), + id: None, + }, + } + } + WalEntryKind::MemoryForget { memory_id } => { + if let Some(mid) = ineru::MemoryId::from_hex(memory_id) { + let mut memory = self.memory.write().await; + match memory.forget(&mid) { + Ok(()) => CortexResponse { + success: true, + detail: None, + id: None, + }, + Err(e) => CortexResponse { + success: false, + detail: Some(format!("MemoryForget failed: {e}")), + id: None, + }, + } + } else { + CortexResponse { + success: false, + detail: Some("Invalid memory ID".to_string()), + id: None, + } + } + } + WalEntryKind::MemoryConsolidate { + consolidated_count: _, + } => { + // Actually perform consolidation on this node + let mut memory = self.memory.write().await; + match memory.consolidate() { + Ok(count) => CortexResponse { + success: true, + detail: Some(count.to_string()), + id: None, + }, + Err(e) => CortexResponse { + success: false, + detail: Some(format!("Consolidation failed: {e}")), + id: None, + }, + } + } + WalEntryKind::LtmEntityCreate { + entity_id: _, + name, + entity_type, + } => { + tracing::debug!(name, entity_type, "Applied LtmEntityCreate"); + CortexResponse { + success: true, + detail: None, + id: None, + } + } + WalEntryKind::LtmLinkCreate { + from_entity, + to_entity, + relation, + weight: _, + } => { + tracing::debug!( + "Applied LtmLinkCreate: {} -> {} ({})", + from_entity, + to_entity, + relation + ); + CortexResponse { + success: true, + detail: None, + id: None, + } + } + WalEntryKind::LtmEntityDelete { entity_id } => { + tracing::debug!(entity_id, "Applied LtmEntityDelete"); + CortexResponse { + success: true, + detail: None, + id: None, + } + } + _ => CortexResponse { + success: true, + detail: None, + id: None, + }, + } + } + + /// Set the last applied log ID. + pub async fn set_last_applied(&self, log_id: LogId) { + let mut la = self.last_applied.write().await; + *la = Some(log_id); + } + + /// Get the last applied log ID. + pub async fn last_applied(&self) -> Option { + let guard = self.last_applied.read().await; + guard.clone() + } + + /// Get the count of applied mutations. + pub async fn applied_count(&self) -> u64 { + *self.applied_count.read().await + } +} + +// ============================================================================ +// RaftStateMachine implementation +// ============================================================================ + +impl RaftStateMachine for Arc { + type SnapshotBuilder = CortexSnapshotBuilder; + + async fn applied_state( + &mut self, + ) -> Result<(Option, StoredMembershipOf), io::Error> { + let la = self.last_applied.read().await; + let membership = self.last_membership.read().await; + Ok((la.clone(), membership.clone())) + } + + async fn apply(&mut self, mut entries: Strm) -> Result<(), io::Error> + where + Strm: futures_util::Stream, io::Error>> + + Unpin + + Send, + { + while let Some(item) = entries.next().await { + let (entry, responder) = item?; + + // Check for membership change + if let Some(membership) = entry.get_membership() { + let mut lm = self.last_membership.write().await; + *lm = StoredMembership::new(Some(entry.log_id.clone()), membership.clone()); + } + + // Apply the business logic + let response = match &entry.payload { + openraft::EntryPayload::Blank => CortexResponse { + success: true, + detail: None, + id: None, + }, + openraft::EntryPayload::Normal(ref req) => { + self.apply_mutation(&req.kind).await + } + openraft::EntryPayload::Membership(_) => CortexResponse { + success: true, + detail: None, + id: None, + }, + }; + + // Update last_applied AFTER successful apply to avoid + // marking entries as applied before they actually are (#1). + { + let mut la = self.last_applied.write().await; + *la = Some(entry.log_id.clone()); + } + + // Send response to client if waiting (leader only) + if let Some(resp) = responder { + resp.send(response); + } + } + + Ok(()) + } + + async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder { + let la = self.last_applied.read().await; + let membership = self.last_membership.read().await; + CortexSnapshotBuilder { + graph: Arc::clone(&self.graph), + memory: Arc::clone(&self.memory), + last_applied: la.clone(), + last_membership: membership.clone(), + } + } + + async fn begin_receiving_snapshot(&mut self) -> Result>, io::Error> { + Ok(Cursor::new(Vec::new())) + } + + async fn install_snapshot( + &mut self, + meta: &SnapshotMetaOf, + snapshot: Cursor>, + ) -> Result<(), io::Error> { + let data = snapshot.into_inner(); + let cluster_snap = ClusterSnapshot::from_bytes(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + // Build both new graph and new memory into temporaries FIRST, + // then swap atomically only if both succeed (#7). + let new_graph = GraphDB::memory() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + for ts in &cluster_snap.triples { + let value = json_to_value(&ts.object); + let triple = aingle_graph::Triple::new( + aingle_graph::NodeId::named(&ts.subject), + aingle_graph::Predicate::named(&ts.predicate), + value, + ); + new_graph + .insert(triple) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; + } + + let new_memory = if !cluster_snap.ineru_ltm.is_empty() { + Some( + IneruMemory::import_snapshot(&cluster_snap.ineru_ltm) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, + format!("Failed to restore Ineru from snapshot: {e}")))? + ) + } else { + None + }; + + // Both built successfully — now swap under both locks so concurrent + // readers never observe new graph with old memory (or vice versa). + let mut graph = self.graph.write().await; + let mut memory = self.memory.write().await; + *graph = new_graph; + if let Some(restored) = new_memory { + *memory = restored; + } + drop(memory); + drop(graph); + + // Update metadata + { + let mut la = self.last_applied.write().await; + *la = meta.last_log_id.clone(); + } + { + let mut lm = self.last_membership.write().await; + *lm = meta.last_membership.clone(); + } + { + let mut snap = self.current_snapshot.write().await; + *snap = Some((meta.clone(), data)); + } + + tracing::info!( + triples = cluster_snap.triples.len(), + "Installed snapshot from leader" + ); + + Ok(()) + } + + async fn get_current_snapshot(&mut self) -> Result>, io::Error> { + let snap = self.current_snapshot.read().await; + match &*snap { + Some((meta, data)) => Ok(Some(Snapshot { + meta: meta.clone(), + snapshot: Cursor::new(data.clone()), + })), + None => Ok(None), + } + } +} + +// ============================================================================ +// Snapshot types +// ============================================================================ + +/// A serializable cluster snapshot for state transfer. +/// +/// When a new node joins the cluster, it receives this snapshot +/// containing the full graph and LTM state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClusterSnapshot { + /// All triples in wire format (subject, predicate, object JSON). + pub triples: Vec, + /// Ineru LTM snapshot (serialized via export_snapshot). + /// STM is NOT replicated — it's node-local working memory. + pub ineru_ltm: Vec, + /// Last applied log index. + pub last_applied_index: u64, + /// Last applied log term. + pub last_applied_term: u64, + /// Blake3 integrity checksum over triples + ineru_ltm. + #[serde(default)] + pub checksum: String, +} + +/// Wire format for a triple in a snapshot. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TripleSnapshot { + pub subject: String, + pub predicate: String, + pub object: serde_json::Value, +} + +impl ClusterSnapshot { + /// Create an empty snapshot. + pub fn empty() -> Self { + Self { + triples: Vec::new(), + ineru_ltm: Vec::new(), + last_applied_index: 0, + last_applied_term: 0, + checksum: String::new(), + } + } + + /// Serialize the snapshot to bytes, computing a blake3 integrity checksum. + pub fn to_bytes(&self) -> Result, String> { + // Serialize everything except checksum first, then patch checksum in + // to avoid cloning the entire snapshot (triples + LTM can be large). + let checksum = compute_checksum(&self.triples, &self.ineru_ltm); + let wrapper = ClusterSnapshotRef { + triples: &self.triples, + ineru_ltm: &self.ineru_ltm, + last_applied_index: self.last_applied_index, + last_applied_term: self.last_applied_term, + checksum: &checksum, + }; + serde_json::to_vec(&wrapper).map_err(|e| format!("Snapshot serialization failed: {e}")) + } + + /// Deserialize a snapshot from bytes, verifying the integrity checksum. + pub fn from_bytes(data: &[u8]) -> Result { + let snap: Self = serde_json::from_slice(data) + .map_err(|e| format!("Snapshot deserialization failed: {e}"))?; + let expected = compute_checksum(&snap.triples, &snap.ineru_ltm); + if !snap.checksum.is_empty() && snap.checksum != expected { + return Err(format!( + "Snapshot checksum mismatch: expected {expected}, got {}", + snap.checksum + )); + } + Ok(snap) + } +} + +// ============================================================================ +// Helpers +// ============================================================================ + +/// Borrow-based snapshot wrapper to avoid cloning during serialization. +#[derive(Serialize)] +struct ClusterSnapshotRef<'a> { + triples: &'a [TripleSnapshot], + ineru_ltm: &'a [u8], + last_applied_index: u64, + last_applied_term: u64, + checksum: &'a str, +} + +/// Compute a blake3 checksum over snapshot content for integrity verification. +fn compute_checksum(triples: &[TripleSnapshot], ineru_ltm: &[u8]) -> String { + let mut hasher = blake3::Hasher::new(); + let triples_bytes = serde_json::to_vec(triples).unwrap_or_default(); + hasher.update(&triples_bytes); + hasher.update(ineru_ltm); + hasher.finalize().to_hex().to_string() +} + +fn json_to_value(v: &serde_json::Value) -> aingle_graph::Value { + match v { + serde_json::Value::String(s) => aingle_graph::Value::String(s.clone()), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + aingle_graph::Value::Integer(i) + } else if let Some(f) = n.as_f64() { + aingle_graph::Value::Float(f) + } else { + aingle_graph::Value::String(n.to_string()) + } + } + serde_json::Value::Bool(b) => aingle_graph::Value::Boolean(*b), + _ => aingle_graph::Value::Json(v.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openraft::vote::RaftLeaderId; + + fn make_graph_and_memory() -> (Arc>, Arc>) { + let graph = GraphDB::memory().unwrap(); + let memory = IneruMemory::agent_mode(); + ( + Arc::new(RwLock::new(graph)), + Arc::new(RwLock::new(memory)), + ) + } + + #[tokio::test] + async fn test_state_machine_new() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(graph, memory); + assert!(sm.last_applied().await.is_none()); + assert_eq!(sm.applied_count().await, 0); + } + + #[tokio::test] + async fn test_apply_triple_insert_real() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(Arc::clone(&graph), Arc::clone(&memory)); + + let kind = WalEntryKind::TripleInsert { + subject: "alice".into(), + predicate: "knows".into(), + object: serde_json::json!("bob"), + triple_id: [0u8; 32], + }; + let resp = sm.apply_mutation(&kind).await; + assert!(resp.success); + assert!(resp.id.is_some(), "TripleInsert should return an ID"); + assert_eq!(sm.applied_count().await, 1); + + // Verify in GraphDB + let g = graph.read().await; + let count = g.count(); + assert!(count >= 1); + } + + #[tokio::test] + async fn test_apply_triple_delete() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(Arc::clone(&graph), Arc::clone(&memory)); + + // Insert a triple first + let triple = aingle_graph::Triple::new( + aingle_graph::NodeId::named("alice"), + aingle_graph::Predicate::named("knows"), + aingle_graph::Value::String("bob".into()), + ); + let tid = { + let g = graph.read().await; + g.insert(triple).unwrap() + }; + + // Delete via state machine + let kind = WalEntryKind::TripleDelete { + triple_id: *tid.as_bytes(), + }; + let resp = sm.apply_mutation(&kind).await; + assert!(resp.success); + } + + #[tokio::test] + async fn test_apply_memory_store() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(graph, memory); + + let kind = WalEntryKind::MemoryStore { + memory_id: "m1".into(), + entry_type: "test".into(), + data: serde_json::json!({"key": "value"}), + importance: 0.8, + }; + let resp = sm.apply_mutation(&kind).await; + assert!(resp.success); + assert!(resp.id.is_some(), "MemoryStore should return an ID"); + } + + #[tokio::test] + async fn test_apply_multiple() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(graph, memory); + for i in 0..5 { + let kind = WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!(i), + triple_id: [i as u8; 32], + }; + sm.apply_mutation(&kind).await; + } + assert_eq!(sm.applied_count().await, 5); + } + + #[tokio::test] + async fn test_apply_ltm_operations() { + let (graph, memory) = make_graph_and_memory(); + let sm = CortexStateMachine::new(graph, memory); + + let resp = sm + .apply_mutation(&WalEntryKind::LtmEntityCreate { + entity_id: "e1".into(), + name: "Entity1".into(), + entity_type: "concept".into(), + }) + .await; + assert!(resp.success); + + let resp = sm + .apply_mutation(&WalEntryKind::LtmLinkCreate { + from_entity: "e1".into(), + to_entity: "e2".into(), + relation: "related_to".into(), + weight: 0.9, + }) + .await; + assert!(resp.success); + + let resp = sm + .apply_mutation(&WalEntryKind::LtmEntityDelete { + entity_id: "e1".into(), + }) + .await; + assert!(resp.success); + + assert_eq!(sm.applied_count().await, 3); + } + + #[tokio::test] + async fn test_install_snapshot_clears_existing_data() { + let (graph, memory) = make_graph_and_memory(); + let sm = Arc::new(CortexStateMachine::new( + Arc::clone(&graph), + Arc::clone(&memory), + )); + + // Pre-populate graph with data that should be cleared + { + let g = graph.read().await; + g.insert(aingle_graph::Triple::new( + aingle_graph::NodeId::named("old_subject"), + aingle_graph::Predicate::named("old_pred"), + aingle_graph::Value::String("old_value".into()), + )) + .unwrap(); + } + assert_eq!(graph.read().await.count(), 1); + + // Create snapshot with different data + let snap = ClusterSnapshot { + triples: vec![TripleSnapshot { + subject: "new_subject".into(), + predicate: "new_pred".into(), + object: serde_json::json!("new_value"), + }], + ineru_ltm: vec![], + last_applied_index: 10, + last_applied_term: 2, + checksum: String::new(), + }; + let data = snap.to_bytes().unwrap(); + + let meta = openraft::storage::SnapshotMeta { + last_log_id: Some(openraft::LogId::new( + openraft::vote::leader_id_adv::CommittedLeaderId::new(2, 0), + 10, + )), + last_membership: openraft::StoredMembership::default(), + snapshot_id: "test".to_string(), + }; + + let mut sm_mut = sm.clone(); + sm_mut + .install_snapshot(&meta, Cursor::new(data)) + .await + .unwrap(); + + // Verify: old data cleared, only snapshot data present + let g = graph.read().await; + assert_eq!(g.count(), 1, "old data should be cleared, only snapshot data remains"); + let triples = g.find(aingle_graph::TriplePattern::any()).unwrap(); + let subject_str = triples[0].subject.to_string(); + assert!( + subject_str.contains("new_subject"), + "Expected subject containing 'new_subject', got '{subject_str}'" + ); + } + + #[test] + fn test_snapshot_empty() { + let snap = ClusterSnapshot::empty(); + assert!(snap.triples.is_empty()); + assert!(snap.ineru_ltm.is_empty()); + assert_eq!(snap.last_applied_index, 0); + } + + #[test] + fn test_snapshot_roundtrip() { + let snap = ClusterSnapshot { + triples: vec![TripleSnapshot { + subject: "alice".into(), + predicate: "knows".into(), + object: serde_json::json!("bob"), + }], + ineru_ltm: vec![1, 2, 3, 4], + last_applied_index: 42, + last_applied_term: 5, + checksum: String::new(), + }; + + let bytes = snap.to_bytes().unwrap(); + let restored = ClusterSnapshot::from_bytes(&bytes).unwrap(); + + assert_eq!(restored.triples.len(), 1); + assert_eq!(restored.triples[0].subject, "alice"); + assert_eq!(restored.ineru_ltm, vec![1, 2, 3, 4]); + assert_eq!(restored.last_applied_index, 42); + assert_eq!(restored.last_applied_term, 5); + } + + #[test] + fn test_snapshot_stm_not_included() { + let snap = ClusterSnapshot::empty(); + let json = serde_json::to_value(&snap).unwrap(); + assert!(json.get("stm").is_none()); + assert!(json.get("ineru_ltm").is_some()); + } + + #[test] + fn test_snapshot_checksum_roundtrip() { + let snap = ClusterSnapshot { + triples: vec![TripleSnapshot { + subject: "alice".into(), + predicate: "knows".into(), + object: serde_json::json!("bob"), + }], + ineru_ltm: vec![10, 20, 30], + last_applied_index: 7, + last_applied_term: 2, + checksum: String::new(), + }; + let bytes = snap.to_bytes().unwrap(); + // Verify checksum was written into serialized data + let raw: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let checksum = raw["checksum"].as_str().unwrap(); + assert!(!checksum.is_empty(), "checksum should be set after to_bytes"); + + // Valid roundtrip succeeds + let restored = ClusterSnapshot::from_bytes(&bytes).unwrap(); + assert_eq!(restored.checksum, checksum); + } + + #[test] + fn test_snapshot_corrupt_data_rejected() { + let snap = ClusterSnapshot { + triples: vec![TripleSnapshot { + subject: "s".into(), + predicate: "p".into(), + object: serde_json::json!("o"), + }], + ineru_ltm: vec![1, 2, 3], + last_applied_index: 1, + last_applied_term: 1, + checksum: String::new(), + }; + let mut bytes = snap.to_bytes().unwrap(); + + // Corrupt one byte in the middle of the payload + let mid = bytes.len() / 2; + bytes[mid] ^= 0xFF; + + // Deserialization should fail (either JSON parse error or checksum mismatch) + let result = ClusterSnapshot::from_bytes(&bytes); + assert!(result.is_err(), "corrupted snapshot must be rejected"); + } + + #[test] + fn test_snapshot_wrong_checksum_rejected() { + // Manually craft a snapshot with a valid structure but wrong checksum + let snap = ClusterSnapshot { + triples: vec![TripleSnapshot { + subject: "a".into(), + predicate: "b".into(), + object: serde_json::json!("c"), + }], + ineru_ltm: vec![], + last_applied_index: 0, + last_applied_term: 0, + checksum: "deadbeef".to_string(), + }; + // Serialize directly (bypassing to_bytes which would compute correct checksum) + let bytes = serde_json::to_vec(&snap).unwrap(); + let result = ClusterSnapshot::from_bytes(&bytes); + assert!(result.is_err()); + assert!( + result.unwrap_err().contains("checksum mismatch"), + "error should mention checksum mismatch" + ); + } + + #[test] + fn test_snapshot_empty_checksum_accepted() { + // Backward compatibility: snapshots without checksum should be accepted + let snap = ClusterSnapshot { + triples: vec![], + ineru_ltm: vec![], + last_applied_index: 0, + last_applied_term: 0, + checksum: String::new(), + }; + let bytes = serde_json::to_vec(&snap).unwrap(); + let result = ClusterSnapshot::from_bytes(&bytes); + assert!(result.is_ok(), "empty checksum should be accepted for backward compat"); + } +} diff --git a/crates/aingle_raft/src/types.rs b/crates/aingle_raft/src/types.rs new file mode 100644 index 0000000..c15fd06 --- /dev/null +++ b/crates/aingle_raft/src/types.rs @@ -0,0 +1,65 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! OpenRaft type configuration for Cortex. + +use aingle_wal::WalEntryKind; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Node identifier. +pub type NodeId = u64; + +/// A Raft client request containing a WAL mutation. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CortexRequest { + pub kind: WalEntryKind, +} + +// Eq is required by openraft; we delegate to PartialEq which is sufficient +// for the WAL entry types used here. +impl Eq for CortexRequest {} + +impl fmt::Display for CortexRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CortexRequest({:?})", std::mem::discriminant(&self.kind)) + } +} + +/// Response from applying a Raft entry. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CortexResponse { + pub success: bool, + pub detail: Option, + /// Generated resource ID (triple hash, memory ID, etc.). + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +impl fmt::Display for CortexResponse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CortexResponse(success={})", self.success) + } +} + +/// Node address information for the cluster. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub struct CortexNode { + pub rest_addr: String, + pub p2p_addr: String, +} + +impl fmt::Display for CortexNode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CortexNode(rest={}, p2p={})", self.rest_addr, self.p2p_addr) + } +} + +// Define the openraft TypeConfig +openraft::declare_raft_types!( + pub CortexTypeConfig: + D = CortexRequest, + R = CortexResponse, + Node = CortexNode, + NodeId = NodeId, +); diff --git a/crates/aingle_viz/Cargo.toml b/crates/aingle_viz/Cargo.toml index d99c3e3..7ed3c63 100644 --- a/crates/aingle_viz/Cargo.toml +++ b/crates/aingle_viz/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_viz" -version = "0.4.2" +version = "0.5.0" description = "DAG Visualization for AIngle - Web-based graph explorer" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -30,8 +30,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" # Graph data -aingle_graph = { version = "0.4", path = "../aingle_graph" } -aingle_minimal = { version = "0.4", path = "../aingle_minimal", default-features = false, features = ["sqlite"] } +aingle_graph = { version = "0.5", path = "../aingle_graph" } +aingle_minimal = { version = "0.5", path = "../aingle_minimal", default-features = false, features = ["sqlite"] } # Utilities log = "0.4" diff --git a/crates/aingle_wal/Cargo.toml b/crates/aingle_wal/Cargo.toml new file mode 100644 index 0000000..a0b1ef8 --- /dev/null +++ b/crates/aingle_wal/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "aingle_wal" +version = "0.5.0" +description = "Write-Ahead Log for AIngle clustering and replication" +license = "Apache-2.0 OR LicenseRef-Commercial" +repository = "https://github.com/ApiliumCode/aingle" +homepage = "https://apilium.com" +documentation = "https://docs.rs/aingle_wal" +authors = ["Apilium Technologies "] +keywords = ["aingle", "wal", "replication", "clustering"] +categories = ["database"] +edition = "2021" +rust-version = "1.83" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +bincode = "2" +blake3 = "1.8" +chrono = { version = "0.4", features = ["serde"] } + +[dev-dependencies] +tempfile = "3.26" diff --git a/crates/aingle_wal/src/entry.rs b/crates/aingle_wal/src/entry.rs new file mode 100644 index 0000000..2951c07 --- /dev/null +++ b/crates/aingle_wal/src/entry.rs @@ -0,0 +1,144 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! WAL entry types and serialization. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +/// A single WAL entry representing one mutation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WalEntry { + /// Monotonically increasing sequence number. + pub seq: u64, + /// Wall-clock timestamp (UTC). + pub timestamp: DateTime, + /// The mutation kind. + pub kind: WalEntryKind, + /// blake3 hash of the previous entry (chain integrity). + pub prev_hash: [u8; 32], + /// blake3 hash of this entry's payload. + pub hash: [u8; 32], +} + +impl WalEntry { + /// Compute the hash for this entry's payload (kind + seq + timestamp + prev_hash). + pub fn compute_hash(seq: u64, timestamp: &DateTime, kind: &WalEntryKind, prev_hash: &[u8; 32]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(&seq.to_le_bytes()); + hasher.update(timestamp.to_rfc3339().as_bytes()); + // Hash the serialized kind + if let Ok(kind_bytes) = serde_json::to_vec(kind) { + hasher.update(&kind_bytes); + } + hasher.update(prev_hash); + *hasher.finalize().as_bytes() + } +} + +/// The kind of mutation recorded in a WAL entry. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum WalEntryKind { + /// Triple inserted into GraphDB. + TripleInsert { + subject: String, + predicate: String, + object: serde_json::Value, + triple_id: [u8; 32], + }, + /// Triple deleted from GraphDB. + TripleDelete { + triple_id: [u8; 32], + }, + /// Memory entry stored in Ineru STM. + MemoryStore { + memory_id: String, + entry_type: String, + data: serde_json::Value, + importance: f32, + }, + /// Memory entry forgotten. + MemoryForget { + memory_id: String, + }, + /// STM → LTM consolidation occurred. + MemoryConsolidate { + consolidated_count: usize, + }, + /// Proof submitted. + ProofSubmit { + proof_id: String, + proof_type: String, + }, + /// Snapshot checkpoint marker. + Checkpoint { + graph_triple_count: usize, + ineru_stm_count: usize, + ineru_ltm_entity_count: usize, + }, + /// LTM entity created (for Ineru replication). + LtmEntityCreate { + entity_id: String, + name: String, + entity_type: String, + }, + /// LTM link created (for Ineru replication). + LtmLinkCreate { + from_entity: String, + to_entity: String, + relation: String, + weight: f32, + }, + /// LTM entity deleted (for Ineru replication). + LtmEntityDelete { + entity_id: String, + }, + /// Serialized openraft Raft log entry. + RaftEntry { + index: u64, + term: u64, + data: Vec, + }, + /// No-op entry for linearizable reads. + Noop, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_entry_kind_serialization() { + let kind = WalEntryKind::TripleInsert { + subject: "alice".into(), + predicate: "knows".into(), + object: serde_json::json!("bob"), + triple_id: [0u8; 32], + }; + let json = serde_json::to_string(&kind).unwrap(); + let back: WalEntryKind = serde_json::from_str(&json).unwrap(); + assert!(matches!(back, WalEntryKind::TripleInsert { .. })); + } + + #[test] + fn test_compute_hash_deterministic() { + let ts = Utc::now(); + let kind = WalEntryKind::TripleDelete { triple_id: [1u8; 32] }; + let prev = [0u8; 32]; + + let h1 = WalEntry::compute_hash(1, &ts, &kind, &prev); + let h2 = WalEntry::compute_hash(1, &ts, &kind, &prev); + assert_eq!(h1, h2); + } + + #[test] + fn test_compute_hash_differs_on_seq() { + let ts = Utc::now(); + let kind = WalEntryKind::TripleDelete { triple_id: [1u8; 32] }; + let prev = [0u8; 32]; + + let h1 = WalEntry::compute_hash(1, &ts, &kind, &prev); + let h2 = WalEntry::compute_hash(2, &ts, &kind, &prev); + assert_ne!(h1, h2); + } +} diff --git a/crates/aingle_wal/src/lib.rs b/crates/aingle_wal/src/lib.rs new file mode 100644 index 0000000..2c1b02c --- /dev/null +++ b/crates/aingle_wal/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Write-Ahead Log (WAL) for AIngle clustering and replication. +//! +//! Provides a durable, ordered log of all mutations before they hit +//! the graph/memory store. Used as the foundation for Raft consensus +//! log replication. + +pub mod entry; +pub mod reader; +pub mod segment; +pub mod writer; + +pub use entry::{WalEntry, WalEntryKind}; +pub use reader::{VerifyResult, WalReader}; +pub use writer::WalWriter; diff --git a/crates/aingle_wal/src/reader.rs b/crates/aingle_wal/src/reader.rs new file mode 100644 index 0000000..472092b --- /dev/null +++ b/crates/aingle_wal/src/reader.rs @@ -0,0 +1,270 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! WAL reader for replay and replication. + +use crate::entry::{WalEntry, WalEntryKind}; +use crate::segment; +use std::io; +use std::path::{Path, PathBuf}; + +/// WAL reader for replay and replication. +pub struct WalReader { + dir: PathBuf, +} + +impl WalReader { + /// Open a WAL directory for reading. + pub fn open(dir: &Path) -> io::Result { + if !dir.exists() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("WAL directory not found: {}", dir.display()), + )); + } + Ok(Self { + dir: dir.to_path_buf(), + }) + } + + /// Read all entries from `start_seq` onwards. + pub fn read_from(&self, start_seq: u64) -> io::Result> { + let segments = segment::list_segments(&self.dir)?; + let mut result = Vec::new(); + + for seg_path in &segments { + let entries = segment::read_entries_from_path(seg_path)?; + for entry in entries { + if entry.seq >= start_seq { + result.push(entry); + } + } + } + + result.sort_by_key(|e| e.seq); + Ok(result) + } + + /// Stream entries from `start_seq` as a Vec (for iteration). + pub fn stream_from(&self, start_seq: u64) -> io::Result> { + self.read_from(start_seq) + } + + /// Verify hash chain integrity across all segments. + pub fn verify_integrity(&self) -> io::Result { + let entries = self.read_from(0)?; + + if entries.is_empty() { + return Ok(VerifyResult { + valid: true, + entries_checked: 0, + first_invalid_seq: None, + }); + } + + // Verify first entry's prev_hash is zeros + if entries[0].prev_hash != [0u8; 32] { + return Ok(VerifyResult { + valid: false, + entries_checked: 1, + first_invalid_seq: Some(entries[0].seq), + }); + } + + // Verify hash chain + for i in 0..entries.len() { + let entry = &entries[i]; + + // Verify this entry's hash + let expected_hash = WalEntry::compute_hash( + entry.seq, + &entry.timestamp, + &entry.kind, + &entry.prev_hash, + ); + if entry.hash != expected_hash { + return Ok(VerifyResult { + valid: false, + entries_checked: i as u64 + 1, + first_invalid_seq: Some(entry.seq), + }); + } + + // Verify chain link + if i > 0 && entry.prev_hash != entries[i - 1].hash { + return Ok(VerifyResult { + valid: false, + entries_checked: i as u64 + 1, + first_invalid_seq: Some(entry.seq), + }); + } + } + + Ok(VerifyResult { + valid: true, + entries_checked: entries.len() as u64, + first_invalid_seq: None, + }) + } + + /// Find the last checkpoint entry. + pub fn last_checkpoint(&self) -> io::Result> { + let entries = self.read_from(0)?; + Ok(entries + .into_iter() + .rev() + .find(|e| matches!(e.kind, WalEntryKind::Checkpoint { .. }))) + } + + /// Count total entries across all segments. + pub fn entry_count(&self) -> io::Result { + let entries = self.read_from(0)?; + Ok(entries.len() as u64) + } +} + +/// Result of WAL integrity verification. +#[derive(Debug, Clone)] +pub struct VerifyResult { + /// Whether the entire WAL is valid. + pub valid: bool, + /// Number of entries checked. + pub entries_checked: u64, + /// First invalid sequence number, if any. + pub first_invalid_seq: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::writer::WalWriter; + + #[test] + fn test_reader_read_from() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + for i in 0..5 { + writer + .append(WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!(i), + triple_id: [i as u8; 32], + }) + .unwrap(); + } + + let reader = WalReader::open(dir.path()).unwrap(); + let entries = reader.read_from(2).unwrap(); + assert_eq!(entries.len(), 3); + assert_eq!(entries[0].seq, 2); + } + + #[test] + fn test_reader_verify_integrity() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + for i in 0..10 { + writer + .append(WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!(i), + triple_id: [i as u8; 32], + }) + .unwrap(); + } + + let reader = WalReader::open(dir.path()).unwrap(); + let result = reader.verify_integrity().unwrap(); + assert!(result.valid); + assert_eq!(result.entries_checked, 10); + assert!(result.first_invalid_seq.is_none()); + } + + #[test] + fn test_reader_empty_wal() { + let dir = tempfile::tempdir().unwrap(); + // Create an empty WAL directory + let _ = WalWriter::open(dir.path()).unwrap(); + + let reader = WalReader::open(dir.path()).unwrap(); + let result = reader.verify_integrity().unwrap(); + assert!(result.valid); + assert_eq!(result.entries_checked, 0); + } + + #[test] + fn test_reader_last_checkpoint() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + writer + .append(WalEntryKind::TripleInsert { + subject: "a".into(), + predicate: "b".into(), + object: serde_json::json!("c"), + triple_id: [0u8; 32], + }) + .unwrap(); + + writer.checkpoint(10, 5, 3).unwrap(); + + writer + .append(WalEntryKind::TripleDelete { + triple_id: [1u8; 32], + }) + .unwrap(); + + let reader = WalReader::open(dir.path()).unwrap(); + let cp = reader.last_checkpoint().unwrap(); + assert!(cp.is_some()); + assert!(matches!( + cp.unwrap().kind, + WalEntryKind::Checkpoint { + graph_triple_count: 10, + .. + } + )); + } + + #[test] + fn test_reader_stream_from() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + for i in 0..3 { + writer + .append(WalEntryKind::MemoryStore { + memory_id: format!("m{}", i), + entry_type: "test".into(), + data: serde_json::json!({"n": i}), + importance: 0.5, + }) + .unwrap(); + } + + let reader = WalReader::open(dir.path()).unwrap(); + let entries = reader.stream_from(0).unwrap(); + assert_eq!(entries.len(), 3); + } + + #[test] + fn test_reader_entry_count() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + for i in 0..7 { + writer + .append(WalEntryKind::TripleDelete { + triple_id: [i as u8; 32], + }) + .unwrap(); + } + + let reader = WalReader::open(dir.path()).unwrap(); + assert_eq!(reader.entry_count().unwrap(), 7); + } +} diff --git a/crates/aingle_wal/src/segment.rs b/crates/aingle_wal/src/segment.rs new file mode 100644 index 0000000..b5bcd1e --- /dev/null +++ b/crates/aingle_wal/src/segment.rs @@ -0,0 +1,283 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! WAL segment file management. +//! +//! WAL is split into segment files of configurable max size. +//! Format per segment: sequence of `[4-byte len][bincode payload]` entries. +//! Filename: `wal-{first_seq:016}.seg` + +use crate::entry::WalEntry; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufReader, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; + +/// Default maximum segment size: 64 MB. +pub const DEFAULT_MAX_SEGMENT_SIZE: u64 = 64 * 1024 * 1024; + +/// A single WAL segment file. +pub struct WalSegment { + path: PathBuf, + file: BufWriter, + first_seq: u64, + last_seq: u64, + size_bytes: u64, +} + +impl WalSegment { + /// Create a new segment file. + pub fn create(dir: &Path, first_seq: u64) -> io::Result { + let filename = format!("wal-{:016}.seg", first_seq); + let path = dir.join(filename); + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&path)?; + Ok(Self { + path, + file: BufWriter::new(file), + first_seq, + last_seq: first_seq, + size_bytes: 0, + }) + } + + /// Open an existing segment file for appending. + pub fn open(path: &Path) -> io::Result { + // Parse first_seq from filename + let filename = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid segment path"))?; + + let first_seq = filename + .strip_prefix("wal-") + .and_then(|s| s.strip_suffix(".seg")) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "invalid segment filename"))?; + + let file = OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + + let size_bytes = file.metadata()?.len(); + + // Read all entries to find last_seq + let mut last_seq = first_seq; + if size_bytes > 0 { + let entries = read_entries_from_path(path)?; + if let Some(last) = entries.last() { + last_seq = last.seq; + } + } + + Ok(Self { + path: path.to_path_buf(), + file: BufWriter::new(file), + first_seq, + last_seq, + size_bytes, + }) + } + + /// Append a WAL entry to the segment. + pub fn append(&mut self, entry: &WalEntry) -> io::Result<()> { + let payload = serde_json::to_vec(entry) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + let len = payload.len() as u32; + self.file.write_all(&len.to_be_bytes())?; + self.file.write_all(&payload)?; + self.size_bytes += 4 + payload.len() as u64; + self.last_seq = entry.seq; + Ok(()) + } + + /// Flush and fsync the segment to disk. + pub fn sync(&mut self) -> io::Result<()> { + self.file.flush()?; + self.file.get_ref().sync_all() + } + + /// Iterate over all entries in this segment. + pub fn iter(&self) -> io::Result> { + read_entries_from_path(&self.path) + } + + /// Current size of the segment file in bytes. + pub fn size(&self) -> u64 { + self.size_bytes + } + + /// The first sequence number in this segment. + pub fn first_seq(&self) -> u64 { + self.first_seq + } + + /// The last sequence number written to this segment. + pub fn last_seq(&self) -> u64 { + self.last_seq + } + + /// Path to the segment file. + pub fn path(&self) -> &Path { + &self.path + } +} + +/// Read all entries from a segment file. +pub fn read_entries_from_path(path: &Path) -> io::Result> { + let file = File::open(path)?; + let file_len = file.metadata()?.len(); + if file_len == 0 { + return Ok(Vec::new()); + } + let mut reader = BufReader::new(file); + let mut entries = Vec::new(); + let mut pos = 0u64; + + loop { + if pos >= file_len { + break; + } + + let mut len_buf = [0u8; 4]; + match reader.read_exact(&mut len_buf) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e), + } + let len = u32::from_be_bytes(len_buf) as usize; + pos += 4; + + let mut payload = vec![0u8; len]; + reader.read_exact(&mut payload)?; + pos += len as u64; + + let entry: WalEntry = serde_json::from_slice(&payload) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + entries.push(entry); + } + + Ok(entries) +} + +/// List all segment files in a directory, sorted by first_seq. +pub fn list_segments(dir: &Path) -> io::Result> { + let mut segments: Vec = std::fs::read_dir(dir)? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.extension().map(|e| e == "seg").unwrap_or(false) + && p.file_name() + .and_then(|n| n.to_str()) + .map(|n| n.starts_with("wal-")) + .unwrap_or(false) + }) + .collect(); + segments.sort(); + Ok(segments) +} + +/// Parse the first_seq from a segment filename. +pub fn parse_segment_seq(path: &Path) -> Option { + path.file_name() + .and_then(|n| n.to_str()) + .and_then(|s| s.strip_prefix("wal-")) + .and_then(|s| s.strip_suffix(".seg")) + .and_then(|s| s.parse().ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::entry::{WalEntry, WalEntryKind}; + use chrono::Utc; + + fn make_entry(seq: u64) -> WalEntry { + let kind = WalEntryKind::TripleInsert { + subject: format!("s{}", seq), + predicate: "p".into(), + object: serde_json::json!("o"), + triple_id: [seq as u8; 32], + }; + let prev_hash = [0u8; 32]; + let ts = Utc::now(); + let hash = WalEntry::compute_hash(seq, &ts, &kind, &prev_hash); + WalEntry { + seq, + timestamp: ts, + kind, + prev_hash, + hash, + } + } + + #[test] + fn test_segment_create_append_iter() { + let dir = tempfile::tempdir().unwrap(); + let mut seg = WalSegment::create(dir.path(), 0).unwrap(); + + for i in 0..5 { + seg.append(&make_entry(i)).unwrap(); + } + seg.sync().unwrap(); + + let entries = seg.iter().unwrap(); + assert_eq!(entries.len(), 5); + assert_eq!(entries[0].seq, 0); + assert_eq!(entries[4].seq, 4); + } + + #[test] + fn test_segment_open() { + let dir = tempfile::tempdir().unwrap(); + + // Create and write + { + let mut seg = WalSegment::create(dir.path(), 10).unwrap(); + seg.append(&make_entry(10)).unwrap(); + seg.append(&make_entry(11)).unwrap(); + seg.sync().unwrap(); + } + + // Re-open + let path = dir.path().join("wal-0000000000000010.seg"); + let seg = WalSegment::open(&path).unwrap(); + assert_eq!(seg.first_seq(), 10); + assert_eq!(seg.last_seq(), 11); + } + + #[test] + fn test_segment_size_limit() { + let dir = tempfile::tempdir().unwrap(); + let mut seg = WalSegment::create(dir.path(), 0).unwrap(); + + seg.append(&make_entry(0)).unwrap(); + seg.sync().unwrap(); + + assert!(seg.size() > 0); + } + + #[test] + fn test_list_segments() { + let dir = tempfile::tempdir().unwrap(); + + // Create multiple segments + for first in [0, 100, 200] { + let mut seg = WalSegment::create(dir.path(), first).unwrap(); + seg.append(&make_entry(first)).unwrap(); + seg.sync().unwrap(); + } + + let segments = list_segments(dir.path()).unwrap(); + assert_eq!(segments.len(), 3); + } + + #[test] + fn test_parse_segment_seq() { + let path = PathBuf::from("wal-0000000000000042.seg"); + assert_eq!(parse_segment_seq(&path), Some(42)); + } +} diff --git a/crates/aingle_wal/src/writer.rs b/crates/aingle_wal/src/writer.rs new file mode 100644 index 0000000..69eff0b --- /dev/null +++ b/crates/aingle_wal/src/writer.rs @@ -0,0 +1,323 @@ +// Copyright 2019-2026 Apilium Technologies OÜ. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 OR Commercial + +//! Thread-safe WAL writer. +//! +//! All writes are serialized through a Mutex. Each write: +//! 1. Assigns next seq number +//! 2. Computes hash chain (prev_hash from last entry) +//! 3. Appends to current segment +//! 4. Calls fsync +//! 5. Rotates segment if size exceeds threshold + +use crate::entry::{WalEntry, WalEntryKind}; +use crate::segment::{self, WalSegment, DEFAULT_MAX_SEGMENT_SIZE}; +use chrono::Utc; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; + +/// Thread-safe WAL writer with hash chain integrity and segment rotation. +pub struct WalWriter { + dir: PathBuf, + current_segment: Mutex, + next_seq: AtomicU64, + last_hash: Mutex<[u8; 32]>, + max_segment_size: u64, +} + +impl WalWriter { + /// Open or create a WAL in the given directory. + pub fn open(dir: &Path) -> io::Result { + std::fs::create_dir_all(dir)?; + + let segments = segment::list_segments(dir)?; + + if segments.is_empty() { + // Fresh WAL + let seg = WalSegment::create(dir, 0)?; + return Ok(Self { + dir: dir.to_path_buf(), + current_segment: Mutex::new(seg), + next_seq: AtomicU64::new(0), + last_hash: Mutex::new([0u8; 32]), + max_segment_size: DEFAULT_MAX_SEGMENT_SIZE, + }); + } + + // Open the last segment + let last_path = segments.last().unwrap(); + let seg = WalSegment::open(last_path)?; + + // Find the last entry to restore state + let entries = seg.iter()?; + let (next_seq, last_hash) = if let Some(last) = entries.last() { + (last.seq + 1, last.hash) + } else { + (seg.first_seq(), [0u8; 32]) + }; + + Ok(Self { + dir: dir.to_path_buf(), + current_segment: Mutex::new(seg), + next_seq: AtomicU64::new(next_seq), + last_hash: Mutex::new(last_hash), + max_segment_size: DEFAULT_MAX_SEGMENT_SIZE, + }) + } + + /// Append a mutation to the WAL. + pub fn append(&self, kind: WalEntryKind) -> io::Result { + let seq = self.next_seq.fetch_add(1, Ordering::SeqCst); + let timestamp = Utc::now(); + + let prev_hash = { + let guard = self.last_hash.lock().unwrap(); + *guard + }; + + let hash = WalEntry::compute_hash(seq, ×tamp, &kind, &prev_hash); + + let entry = WalEntry { + seq, + timestamp, + kind, + prev_hash, + hash, + }; + + { + let mut seg = self.current_segment.lock().unwrap(); + seg.append(&entry)?; + seg.sync()?; + + // Rotate if needed + if seg.size() >= self.max_segment_size { + let new_seq = self.next_seq.load(Ordering::SeqCst); + let new_seg = WalSegment::create(&self.dir, new_seq)?; + *seg = new_seg; + } + } + + // Update last_hash + { + let mut guard = self.last_hash.lock().unwrap(); + *guard = entry.hash; + } + + Ok(entry) + } + + /// Flush the current segment to disk. + pub fn sync(&self) -> io::Result<()> { + let mut seg = self.current_segment.lock().unwrap(); + seg.sync() + } + + /// The next sequence number that will be assigned. + pub fn last_seq(&self) -> u64 { + let next = self.next_seq.load(Ordering::SeqCst); + if next == 0 { 0 } else { next - 1 } + } + + /// Get the WAL directory path. + pub fn dir(&self) -> &Path { + &self.dir + } + + /// Write a checkpoint entry. + pub fn checkpoint( + &self, + graph_triple_count: usize, + ineru_stm_count: usize, + ineru_ltm_entity_count: usize, + ) -> io::Result { + self.append(WalEntryKind::Checkpoint { + graph_triple_count, + ineru_stm_count, + ineru_ltm_entity_count, + }) + } + + /// Truncate WAL entries before `seq` by removing old segment files. + pub fn truncate_before(&self, seq: u64) -> io::Result { + let segments = segment::list_segments(&self.dir)?; + let mut removed = 0; + + for seg_path in &segments { + if segment::parse_segment_seq(seg_path).is_some() { + // Only remove segments whose entries are all before `seq` + let entries = segment::read_entries_from_path(seg_path)?; + if let Some(last) = entries.last() { + if last.seq < seq { + std::fs::remove_file(seg_path)?; + removed += 1; + } + } + } + } + + Ok(removed) + } + + /// Get WAL statistics. + pub fn stats(&self) -> io::Result { + let segments = segment::list_segments(&self.dir)?; + let total_size: u64 = segments + .iter() + .filter_map(|p| std::fs::metadata(p).ok()) + .map(|m| m.len()) + .sum(); + + Ok(WalStats { + segment_count: segments.len(), + total_size_bytes: total_size, + last_seq: self.last_seq(), + next_seq: self.next_seq.load(Ordering::SeqCst), + }) + } +} + +/// WAL statistics. +#[derive(Debug, Clone)] +pub struct WalStats { + pub segment_count: usize, + pub total_size_bytes: u64, + pub last_seq: u64, + pub next_seq: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_writer_append_and_seq() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + let e1 = writer + .append(WalEntryKind::TripleInsert { + subject: "a".into(), + predicate: "b".into(), + object: serde_json::json!("c"), + triple_id: [0u8; 32], + }) + .unwrap(); + assert_eq!(e1.seq, 0); + + let e2 = writer + .append(WalEntryKind::TripleDelete { + triple_id: [1u8; 32], + }) + .unwrap(); + assert_eq!(e2.seq, 1); + assert_eq!(e2.prev_hash, e1.hash); + } + + #[test] + fn test_writer_reopen() { + let dir = tempfile::tempdir().unwrap(); + + // Write some entries + { + let writer = WalWriter::open(dir.path()).unwrap(); + for i in 0..3 { + writer + .append(WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!("o"), + triple_id: [i as u8; 32], + }) + .unwrap(); + } + } + + // Reopen and continue + let writer = WalWriter::open(dir.path()).unwrap(); + assert_eq!(writer.last_seq(), 2); + + let e = writer + .append(WalEntryKind::TripleDelete { + triple_id: [99u8; 32], + }) + .unwrap(); + assert_eq!(e.seq, 3); + } + + #[test] + fn test_hash_chain_integrity() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + let mut entries = Vec::new(); + for i in 0..5 { + let e = writer + .append(WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!(i), + triple_id: [i as u8; 32], + }) + .unwrap(); + entries.push(e); + } + + // Verify chain + for i in 1..entries.len() { + assert_eq!(entries[i].prev_hash, entries[i - 1].hash); + } + } + + #[test] + fn test_checkpoint() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + let cp = writer.checkpoint(100, 50, 25).unwrap(); + assert!(matches!(cp.kind, WalEntryKind::Checkpoint { .. })); + } + + #[test] + fn test_stats() { + let dir = tempfile::tempdir().unwrap(); + let writer = WalWriter::open(dir.path()).unwrap(); + + writer + .append(WalEntryKind::TripleDelete { + triple_id: [0u8; 32], + }) + .unwrap(); + + let stats = writer.stats().unwrap(); + assert_eq!(stats.segment_count, 1); + assert!(stats.total_size_bytes > 0); + } + + #[test] + fn test_truncate_before() { + let dir = tempfile::tempdir().unwrap(); + + // Create first segment with entries 0-2 + { + let writer = WalWriter::open(dir.path()).unwrap(); + for i in 0..3 { + writer + .append(WalEntryKind::TripleInsert { + subject: format!("s{}", i), + predicate: "p".into(), + object: serde_json::json!(i), + triple_id: [i as u8; 32], + }) + .unwrap(); + } + } + + // Truncate shouldn't remove the only segment since entries aren't all < seq + let writer = WalWriter::open(dir.path()).unwrap(); + let removed = writer.truncate_before(1).unwrap(); + assert_eq!(removed, 0); // segment has entries 0,1,2 — last (2) >= 1 + } +} diff --git a/crates/aingle_zk/Cargo.toml b/crates/aingle_zk/Cargo.toml index 5b7e352..42cdd25 100644 --- a/crates/aingle_zk/Cargo.toml +++ b/crates/aingle_zk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aingle_zk" -version = "0.4.2" +version = "0.5.0" description = "Zero-Knowledge Proofs for AIngle - privacy-preserving cryptographic primitives" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" diff --git a/crates/ineru/Cargo.toml b/crates/ineru/Cargo.toml index e28d6f7..42be4db 100644 --- a/crates/ineru/Cargo.toml +++ b/crates/ineru/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ineru" -version = "0.4.2" +version = "0.5.0" description = "Ineru: Neural-inspired memory system for AIngle AI agents" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" diff --git a/crates/kaneru/Cargo.toml b/crates/kaneru/Cargo.toml index 146cef3..58de0ed 100644 --- a/crates/kaneru/Cargo.toml +++ b/crates/kaneru/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kaneru" -version = "0.4.2" +version = "0.5.0" description = "Kaneru: Unified Multi-Agent Execution System for AIngle AI agents" license = "Apache-2.0 OR LicenseRef-Commercial" repository = "https://github.com/ApiliumCode/aingle" @@ -31,7 +31,7 @@ serde_json = "1.0" log = "0.4" # AI Memory integration -ineru = { version = "0.4", path = "../ineru", optional = true } +ineru = { version = "0.5", path = "../ineru", optional = true } # Random for exploration (updated from 0.7) rand = { version = "0.9", default-features = false, features = ["std", "thread_rng"] }