diff --git a/CHANGELOG.md b/CHANGELOG.md index 671eb066..b76e4170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ the release workflow refuses a tag without one and copies it onto the GitHub release. The crates are versioned together, so one number is the version of all of them. +## 0.0.2 + +Every crate now has its own README, so its page on crates.io says what it is. In 0.0.1 only `zudb` and `zudb-cli` had one, and both showed the repository README. There are no code changes in this release. + +- Each `zudb-*` crate has a README covering what it does, how it is put together, and a short example against its real API. Each also says the crate is published for `zudb` to depend on, and that the import keeps the workspace name (`use zu_query` for `zudb-query`). +- `zudb` has a README of its own with the quickstart, databases and connections, statements, transactions, bulk loading, errors, and the storage engines. Its Rust examples are programs in `crates/zu-snippets`, and the snippet test compiles and runs them against the README. +- `zudb-cli` has a README covering the commands of the `zu` binary, the shell, output formats, and exit codes. + ## 0.0.1 The first release, and the first time zu is on crates.io. It is early: diff --git a/Cargo.toml b/Cargo.toml index 0fe7c254..ed83b3d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] exclude = ["fuzz"] [workspace.package] -version = "0.0.1" +version = "0.0.2" edition = "2024" rust-version = "1.98" license = "Apache-2.0" @@ -12,19 +12,19 @@ repository = "https://github.com/tamnd/zu" authors = ["Tam Nguyen "] [workspace.dependencies] -zu-common = { package = "zudb-common", path = "crates/zu-common", version = "=0.0.1" } -zu-encoding = { package = "zudb-encoding", path = "crates/zu-encoding", version = "=0.0.1" } -zu-json = { package = "zudb-json", path = "crates/zu-json", version = "=0.0.1" } -zu-vector = { package = "zudb-vector", path = "crates/zu-vector", version = "=0.0.1" } -zu-storage = { package = "zudb-storage", path = "crates/zu-storage", version = "=0.0.1" } -zu-zu1 = { package = "zudb-zu1", path = "crates/zu-zu1", version = "=0.0.1" } -zu-sqlite = { package = "zudb-sqlite", path = "crates/zu-sqlite", version = "=0.0.1" } -zu-s3 = { package = "zudb-s3", path = "crates/zu-s3", version = "=0.0.1" } -zu-query = { package = "zudb-query", path = "crates/zu-query", version = "=0.0.1" } -zu-exec = { package = "zudb-exec", path = "crates/zu-exec", version = "=0.0.1" } -zu = { package = "zudb", path = "crates/zu", version = "=0.0.1" } -zu-arrow = { package = "zudb-arrow", path = "crates/zu-arrow", version = "=0.0.1" } -zu-corpus = { package = "zudb-corpus", path = "crates/zu-corpus", version = "=0.0.1" } +zu-common = { package = "zudb-common", path = "crates/zu-common", version = "=0.0.2" } +zu-encoding = { package = "zudb-encoding", path = "crates/zu-encoding", version = "=0.0.2" } +zu-json = { package = "zudb-json", path = "crates/zu-json", version = "=0.0.2" } +zu-vector = { package = "zudb-vector", path = "crates/zu-vector", version = "=0.0.2" } +zu-storage = { package = "zudb-storage", path = "crates/zu-storage", version = "=0.0.2" } +zu-zu1 = { package = "zudb-zu1", path = "crates/zu-zu1", version = "=0.0.2" } +zu-sqlite = { package = "zudb-sqlite", path = "crates/zu-sqlite", version = "=0.0.2" } +zu-s3 = { package = "zudb-s3", path = "crates/zu-s3", version = "=0.0.2" } +zu-query = { package = "zudb-query", path = "crates/zu-query", version = "=0.0.2" } +zu-exec = { package = "zudb-exec", path = "crates/zu-exec", version = "=0.0.2" } +zu = { package = "zudb", path = "crates/zu", version = "=0.0.2" } +zu-arrow = { package = "zudb-arrow", path = "crates/zu-arrow", version = "=0.0.2" } +zu-corpus = { package = "zudb-corpus", path = "crates/zu-corpus", version = "=0.0.2" } thiserror = "2" crc32c = "0.6" diff --git a/conformance.toml b/conformance.toml index 97a1440e..66cd52d1 100644 --- a/conformance.toml +++ b/conformance.toml @@ -9,7 +9,7 @@ [engine] name = "zu" -version = "0.0.1" +version = "0.0.2" # What the storage can hold. [data] diff --git a/crates/zu-arrow/Cargo.toml b/crates/zu-arrow/Cargo.toml index f382a6d7..d57ce1ee 100644 --- a/crates/zu-arrow/Cargo.toml +++ b/crates/zu-arrow/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-arrow/README.md b/crates/zu-arrow/README.md new file mode 100644 index 00000000..c3630691 --- /dev/null +++ b/crates/zu-arrow/README.md @@ -0,0 +1,67 @@ +# zudb-arrow + +Turns a zu query result into Arrow arrays. This is the fast way to get data out of zu: a result becomes one Arrow buffer per column instead of one object per row, and pandas, polars, DuckDB, and the JavaScript dataframe libraries can read it without copying it again. + +## How it works + +The engine does most of the work before this crate sees anything. On a plain projection the sink in `zudb-exec` fills column buffers, not rows, and they are already in the layout Arrow uses: values end to end, a validity bitmap that is left out when nothing is null, and strings as bytes plus offsets. `zudb-query` exposes them through `QueryResult::columnar` (borrowed) and `QueryResult::into_columns` (owned). + +This crate puts an Arrow type and a bitmap around those buffers. + +- `Table::of(&result, &tables)` borrows the result, so the buffers are copied once into arrays. +- `Table::taken(result, &tables)` consumes the result and moves the buffers in. For integers, floats, booleans, strings, dates, times, datetimes, and durations this costs a pointer, not a copy. If the sink built rows instead of columns, it quietly falls back to `of`. +- Nodes, rels, paths, lists, and records have no buffer to move. They are built by hand as Arrow structs and lists in the `values` module. +- A `Table` has a schema, one array per column, and a row count. `batches(n)` cuts it into `RecordBatch`es by slicing, which copies nothing. An empty result still gives one empty batch, so the reader learns the schema. + +Node and rel values carry a table id. To turn that into a name, pass something that implements the `Tables` trait. `()` knows no names and is fine when the result has no node or rel columns. + +A column has one type, which the engine decides. Two things are refused here because Arrow has no type for them: a time with a UTC offset, and a handle to a graph or a binding table. Errors come back as `Error::Type`, `Error::Value`, or `Error::Arrow`. + +The code lives with the engine rather than in a client because every client should convert a column the same way. The Python and JavaScript clients both use it. + +## Using it + +```rust +use zu_arrow::{BATCH, Table}; + +fn main() -> Result<(), Box> { + let db = zudb::Database::memory()?; + let mut conn = db.connect()?; + conn.execute("INSERT (p:person {uid: 1, name: 'ada'})")?; + conn.execute("INSERT (p:person {uid: 2, name: 'grace'})")?; + + let result = conn.query("MATCH (p:person) RETURN p.name AS name, p.uid AS uid")?; + + // `taken` consumes the result and moves its column buffers into + // Arrow. `()` is the table-name lookup, which only node and rel + // columns need. + let table = Table::taken(result, &())?; + println!("{} rows, schema {:?}", table.rows(), table.schema()); + for batch in table.batches(BATCH) { + let batch = batch?; + println!("{} columns, {} rows", batch.num_columns(), batch.num_rows()); + } + Ok(()) +} +``` + +That prints a schema with `name` as `Utf8` and `uid` as `Int64`, and one batch of two rows. `BATCH` is the default batch size, 65,536 rows. + +## Features + +Both features are off by default. + +- `ffi` adds `Table::into_stream`, `stream`, and `stream_taken`, which return an `FFI_ArrowArrayStream`. That is the Arrow C Data Interface, the way to hand a result to pyarrow, DuckDB, Go, or the JVM in the same process. The buffers are shared with the reader and released when it is done. +- `ipc` adds `Table::into_ipc`, `ipc`, and `ipc_taken`, which return the bytes of an Arrow IPC stream. That is for runtimes that cannot read a C struct, which in practice means JavaScript. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-arrow` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_arrow`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-cli/Cargo.toml b/crates/zu-cli/Cargo.toml index 096d2c8a..d44a3c1c 100644 --- a/crates/zu-cli/Cargo.toml +++ b/crates/zu-cli/Cargo.toml @@ -7,7 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true -readme = "../../README.md" +readme = "README.md" keywords = ["database", "graph", "gql", "cli"] categories = ["command-line-utilities", "database-implementations"] diff --git a/crates/zu-cli/README.md b/crates/zu-cli/README.md new file mode 100644 index 00000000..e6342faf --- /dev/null +++ b/crates/zu-cli/README.md @@ -0,0 +1,95 @@ +# zudb-cli + +The `zu` command-line tool for [zu](https://github.com/tamnd/zu), an embedded property-graph database. It loads data into `.zu1` files, runs queries against them, checks them, and gives you an interactive shell and a language server for editors. + +``` +cargo install zudb-cli +``` + +That installs one binary, `zu`. The package is named `zudb-cli` because `zu` was already taken on crates.io. Reading Parquet edge lists needs the `arrow` feature: `cargo install zudb-cli --features arrow`. + +zu is early and not ready for data you care about. Flags and output can change between 0.0.x releases. + +## A first session + +``` +zu copy edges.csv graph.zu1 +zu stat graph.zu1 +zu query graph.zu1 -c 'MATCH (n:node) RETURN count(n) AS n' +zu query graph.zu1 -c 'MATCH (n:node {id: $id}) RETURN n.id' -p id=10 --format json +zu shell graph.zu1 +``` + +`zu copy` builds a new file from an edge list, `zu query` runs one statement and exits, and `zu shell` keeps the file open so the catalog, statistics, plan cache and decoded blocks are paid for once. + +## Commands + +| Command | What it does | +| --- | --- | +| `zu shell []` | open an interactive session on a file, or on nothing | +| `zu query -c ` | run one statement and print the result | +| `zu lsp --stdio [--db ]` | speak the language server protocol, for an editor | +| `zu copy ` | bulk load an edge list, or a whole dataset, into a new file | +| `zu convert ` | rewrite an edge list in another format, or a database in another engine | +| `zu verify ` | walk every checksum and cross-check the structure | +| `zu stat ` | print the size breakdown: schema, free space, and data | +| `zu analyze ` | rebuild every rel table's optimizer summary | +| `zu neighbors [--in] [--key] ` | print one node's neighbor list, in either direction | +| `zu lookup ` | resolve an original id through the primary-key index | +| `zu edge [--in] ` | ask whether one edge exists, without decoding the list | +| `zu conformance ...` | declare, verify, tally, and score conformance reports | +| `zu corpus ` | run the shared corpus cases against this build | +| `zu version` | print the version, the C ABI revision, and what this build supports | + +`zu help` lists them and `zu help ` (or `zu --help`) prints the synopsis, examples and related commands for one. All of it is generated from one table in the source, so the usage line you get after a wrong command line is the same one the help shows. + +### Loading and converting + +`zu copy` takes an edge list as whitespace separated text, CSV, or Parquet, and writes one node table and one rel table. `--nodes nodes.csv` adds node properties, and `--reorder degree|bfs|none` renumbers nodes before writing so that adjacency compresses better. Original ids stay readable. For labelled data, give one file per table: `zu copy --node Account=nodes/Account.csv --rel transfer=Account:Account:rels/transfer.csv fin.zu1`. + +`zu convert` moves an edge list between text, CSV and Parquet, or a database between the `zu1` format and SQLite (`graph.zu1` to `graph.db` and back). Both engines number rows the same way, so the converted file answers queries with the same ids. + +### The shell + +At a terminal, `zu shell` is an editor with history, multi-line statements, syntax highlighting, and tab completion that offers the labels and properties that are actually in the open file. Backslash commands answer questions about the file without going through the query language: + +| Command | What it shows | +| --- | --- | +| `\d [NAME]` | the tables in the file, or one table's columns | +| `\l` | the graphs in the file | +| `\session` | the schema, graph, time zone and parameters of this session | +| `\i FILE` | runs the statements in a file | +| `\timing [on\|off]` | how long each statement took | +| `\?` and `\q` | help, and quit | + +When standard input is not a terminal, or with `--format jsonl`, the shell speaks a line protocol for programs instead. The first line out is a greeting with the protocol and build versions. Each line in is either a bare statement or a JSON frame such as `{"op":"query","q":"...","params":{...}}`, and the other ops are `prepare`, `execute`, `close_stmt`, `explain`, `explain_analyze`, `hello` and `quit`. Each reply is one JSON object with the GQLSTATUS, columns and rows, or an error. A failed statement does not end the session. The protocol is described in `docs/api/jsonl-protocol.md` in the repository. + +### Editors + +`zu lsp --stdio` is a language server in the same binary. Completion comes from the catalog of the file passed with `--db`, diagnostics come from the same parser that would reject the statement, and highlighting uses the same scanner as the shell prompt. + +## Output and exit codes + +Commands that print results take `--format`. `query` takes `table` or `json`. `stat`, `verify`, `convert`, `copy` and `version` take `text` or `json`, and `zu help --format json` prints the whole command table for tools. The flag is spelled and read the same way by every command. + +The exit code depends on the kind of error, not on which command hit it: + +| Code | Meaning | +| --- | --- | +| 0 | success | +| 1 | the statement failed with a GQL condition | +| 2 | bad usage or an invalid argument | +| 3 | an I/O error | +| 4 | the file is corrupt, or was written by something this build cannot read | +| 5 | lost a write conflict to another writer | +| 130 | interrupted | + +## How it is built + +The binary is a thin layer over [`zudb`](https://crates.io/crates/zudb). Every query goes through the same `Session` the Rust API uses, so the CLI and an embedding program get the same answers. Argument parsing is hand-written rather than clap, because the command surface is small and the binary has a 15 MiB size budget. JSON reading and writing come from the small `zudb-json` crate for the same reason. The shell is split into a terminal driver (`term`), key decoding (`keys`), a line editor (`line`) and the loop that joins them (`repl`), so everything except the loop is tested without a terminal. The `corpus` and `conformance` commands are the test harnesses the release process runs, and they use `zudb-corpus`. + +To use zu from Rust code instead of the shell, depend on [`zudb`](https://crates.io/crates/zudb). + +## License + +Apache-2.0 diff --git a/crates/zu-common/Cargo.toml b/crates/zu-common/Cargo.toml index fa08c3e7..a3802a4b 100644 --- a/crates/zu-common/Cargo.toml +++ b/crates/zu-common/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-common/README.md b/crates/zu-common/README.md new file mode 100644 index 00000000..b8fef8bb --- /dev/null +++ b/crates/zu-common/README.md @@ -0,0 +1,79 @@ +# zudb-common + +The shared vocabulary of [zu](https://github.com/tamnd/zu), an embedded property-graph database: ids, the error type, the type model, and the value types (decimals, dates and times, byte strings) that every other crate in the workspace needs to agree on. + +It is the bottom of the dependency graph. It depends only on `thiserror`, and every other zu crate depends on it. + +## What is in it + +| Item | What it is | +| --- | --- | +| `NodeId`, `RelId`, `TableId`, `NodeGroupId`, `NodeOffset`, `Epoch` | Graph identifiers. `NodeId` packs a table, a node group and a row into one `u64`. | +| `ZuError`, `Result` | The error every zu crate returns. Query errors carry a GQLSTATUS diagnostic. | +| `gqlstatus` | The ISO/IEC 39075 condition codes, generated from the standard's own table, and the `DiagnosticRecord` that carries one back to a caller. | +| `types` | The two layer type model: `LogicalType` is what the language talks about, `PhysicalType` is what a vector holds. | +| `Decimal` | Exact decimals as an `i128` and a scale, up to 38 digits. | +| `temporal` | Dates, times, datetimes and durations, stored as plain counts (days, nanoseconds, months). | +| `bytes`, `unicode`, `keywords` | Byte string literals, Unicode normalization (NFC, NFD, NFKC, NFKD), and the GQL reserved word lists. | +| `Interrupt` | A handle a caller keeps on a running statement to stop it or read its progress. | +| `IdMap`, `IdSet` | Hash maps and sets with a cheap hasher for keys that come from the catalog, not from users. | + +## Ids + +`NodeId` is the one layout that is written to disk by every storage engine, so it does not change between versions: + +```text +bits 63..50 table_id (14 bits, up to 16 384 tables) +bits 49..28 node_group (22 bits, up to 4 194 304 groups per table) +bits 27..11 row (17 bits, GROUP_ROWS = 131 072 rows per group) +bits 10..0 reserved (must be zero) +``` + +A node group is the unit storage works in: a table is split into groups of `GROUP_ROWS` rows, and each group's columns are stored and compressed together. The full data model is in `docs/03-data-model.md`. + +## Values + +The value types are built so that comparing two values is comparing two integers. A date is a day count since 1970-01-01, a local time is nanoseconds since midnight, and a duration is either a month count or a nanosecond count depending on its kind. Calendar arithmetic only happens when text is parsed or printed. `Decimal` works the same way: `1.20` and `1.2` compare equal, and each prints back the way it was written. + +## Errors + +`ZuError` has a variant for I/O, corrupt data, unsupported format ids, invalid arguments, write conflicts and interrupts. Errors raised by a query are `ZuError::Gql` and carry a GQLSTATUS code, a severity, and optionally a position in the statement text, so a client can point at the token that failed instead of parsing the message. + +## Example + +```rust +use zu_common::gqlstatus::codes; +use zu_common::{Decimal, LogicalType, NodeId, Temporal, ZuError}; + +fn main() { + // A node id packs the table, the node group and the row into one u64. + let id = NodeId::new(3, 7, 42); + assert_eq!((id.table(), id.group(), id.row()), (3, 7, 42)); + + // Decimals are exact: an i128 and a scale, never a float. + let price = Decimal::parse("19.99", 2).unwrap(); + let tax = Decimal::parse("1.60", 2).unwrap(); + println!("{}", price.add(&tax).unwrap()); // 21.59 + + // A date is a day count, so comparing two dates compares two integers. + let day = Temporal::parse(&LogicalType::Date, "2026-10-02").unwrap(); + println!("{day}"); + + // Errors carry a GQLSTATUS code from ISO/IEC 39075. + let err = ZuError::gql(codes::C22012, "x / 0"); + assert_eq!(err.gqlstatus().map(|s| s.code()), Some("22012")); + println!("{err}"); +} +``` + +## Should you depend on this? + +Probably not directly. This crate is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and all the zu crates are released together with exact version pins between them. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-common` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_common`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-corpus/Cargo.toml b/crates/zu-corpus/Cargo.toml index bb3fc94e..82ae509c 100644 --- a/crates/zu-corpus/Cargo.toml +++ b/crates/zu-corpus/Cargo.toml @@ -6,6 +6,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" description = "The cross-client conformance corpus and its Rust runner" # Published as zudb-*, because zu is taken on crates.io, but the diff --git a/crates/zu-corpus/README.md b/crates/zu-corpus/README.md new file mode 100644 index 00000000..5182895d --- /dev/null +++ b/crates/zu-corpus/README.md @@ -0,0 +1,79 @@ +# zudb-corpus + +The cross-client conformance corpus for [zu](https://github.com/tamnd/zu), and the Rust runner that checks the engine against it. + +zu has clients in several languages. Some link the engine and some go through the C ABI, and each decodes values with its own code. The corpus is how we check that a value written through one client and read through another means the same thing. It is a set of YAML files, each one a list of statements and what they must return, versioned with the engine and shipped as a release artifact (`conformance-.tar.zst`) that every client repository runs. This crate reads those files and runs them against the engine in this repository. The runners for the other clients live in [tamnd/zu-kit](https://github.com/tamnd/zu-kit). + +## What a case looks like + +```yaml +schema: 4 +suite: boolean +doc: The truth values and the operators over them. + +cases: + - name: and-with-a-falsehood + doc: One false makes the conjunction false whichever side it is on. + query: RETURN false AND true AS n + columns: + - n + rows: + - values: + - type: BOOL + value: false + + - name: a-write-clause-that-is-reserved + doc: CREATE is not in the v0 core and the parser says so. + query: CREATE (n:person) + raises: "42001" +``` + +A case expects either `columns` and `rows` in order, or `raises` with a five character GQLSTATUS code. It names the code and not the message, because the code is the contract. Every value is written as `{type, value}` with the GQL type spelled out, so no reader has to guess whether `1` is an integer and how wide it is. The value is a plain YAML scalar only where that is exact. INT64 and UINT64, decimals, floats and temporal values are written as strings, because many YAML readers turn every number into a double, and an integer written as a bare number is refused rather than read loosely. A suite can start with a `load:` block, a small table of typed columns plus edges that each runner puts into a fresh database through its own bulk load path before the cases run. A case can also give `setup` statements, run statements on more than one named connection to test transactions, and describe the Arrow schema its result must export as. + +The cases are in `conformance/cases/` in the repository. + +## Running it + +```rust +use std::path::Path; + +fn main() -> Result<(), String> { + let suites = zu_corpus::load(Path::new("conformance/cases"))?; + let scratch = std::env::temp_dir().join("zu-corpus-run"); + std::fs::create_dir_all(&scratch).map_err(|e| e.to_string())?; + + let report = zu_corpus::run(&suites, &scratch); + for ran in report.failures() { + println!("{ran}"); + } + println!("{}", report.summary()); + Ok(()) +} +``` + +From the repository you can also run `cargo test -p zudb-corpus`, or `zu corpus conformance/cases` through the shipped binary. Add `--strict` to fail on unsupported cases too. + +Each case gets a fresh database file under the directory you pass, named after the suite and the case, so a failure leaves a file you can open. A case comes back as `Passed`, `Failed`, or `Unsupported`. Unsupported means the engine refused the statement with a syntax error or a feature-not-supported condition (GQLSTATUS class `42` or `0A`). The corpus allows that on purpose: cases can be written ahead of the engine, and the engine catches up. A failed setup step is never counted as a pass. + +## How it is organized + +| Module | What it does | +| --- | --- | +| `yaml` | reads the small subset of YAML the cases use: block mappings, block sequences, one-line scalars, two space indents. Anything else is an error with a line number. | +| `value` | the `{type, value}` encoding, and comparing a decoded value with what the engine returned | +| `case` | `Suite`, `Case` and `Expect`, and parsing a file into them | +| `load` | the data a suite loads before its cases, and loading it into a database | +| `arrow` | checks the Arrow export of a result: the column types, and refusals such as a time with an offset. Behind the default `arrow` feature, using `zudb-arrow`. | +| `runner` | `run`, which executes every case and returns a `Report` with one `Ran` per case and a one-line `summary()` | + +`load(dir)` reads every `.yaml` file in a directory in sorted order, so two machines produce reports that line up line by line. It also checks that the `suite:` name in each file matches the file name. + +## Should you depend on this? + +Probably not directly. This crate is one part of zu, published so that [`zudb-cli`](https://crates.io/crates/zudb-cli) can depend on it for `zu corpus`. Its API changes whenever zu needs it to, with no deprecation period, and the versions of all the zu crates move together. To use zu from Rust, add [`zudb`](https://crates.io/crates/zudb). + +The package is named `zudb-corpus` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_corpus`. + +## License + +Apache-2.0 diff --git a/crates/zu-encoding/Cargo.toml b/crates/zu-encoding/Cargo.toml index a8987fd9..96b44b32 100644 --- a/crates/zu-encoding/Cargo.toml +++ b/crates/zu-encoding/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-encoding/README.md b/crates/zu-encoding/README.md new file mode 100644 index 00000000..7923f21f --- /dev/null +++ b/crates/zu-encoding/README.md @@ -0,0 +1,88 @@ +# zudb-encoding + +The lightweight compression schemes that [zu](https://github.com/tamnd/zu) uses for columns on disk, and the cascade that picks one for each segment. + +Each scheme is a pair of plain functions, `encode(values, &mut out)` and `decode(bytes, max_values, &mut out)`, over byte buffers. There is no I/O and no state. The goal for every decoder is at least 1 GB/s per core with no more than 64 KiB of scratch memory, which is why these are lightweight encodings and not general-purpose compression. + +## Encodings + +Every encoding has a stable one-byte `EncodingId`. It is written into the file in front of each segment, so the numbers never change once released. + +| Id | Encoding | Module | Used for | +| --- | --- | --- | --- | +| 0 | Plain | `segment` | Fallback when nothing else is smaller | +| 1 | Constant | `segment` | A segment where every value is the same | +| 2 | RLE | `rle` | Long runs of repeated values | +| 3 | Dict | `dict` | Few distinct values (up to 8192), sorted so range filters stay cheap | +| 4 | Frame of reference + bit packing | `for_bitpack` | Values in a narrow range | +| 5 | Delta + bit packing | `delta` | Sorted or slowly growing values, like ids | +| 6 | ALP | `alp` | Doubles that are really decimals, like prices or coordinates | +| 7 | ALP_RD | `alp_rd` | Doubles that use their full mantissa | +| 8 | FSST | `fsst` | Strings, using a trained table of common substrings | +| 9 | Bool bit packing | `bool_bitpack` | 0 and 1 values, one bit each | +| 10 | Frequency | `frequency` | One dominant value and scattered exceptions | +| 11 | Zstd | `zstd_leaf` | Cold string segments where ratio matters more than speed | +| 12 | Delta + patched bit packing | `delta_patch` | Adjacency lists, where a rare large gap would otherwise widen every value | + +Underneath them are a few building blocks: `bitpack` packs 1024 values at a fixed width in a FastLanes-style interleaved layout that the compiler auto-vectorizes, `patch` stores outliers as exceptions so they do not set the width of a whole chunk, and `validity` stores null bitmaps separately from values. + +## The cascade + +`segment::encode_auto` is what the storage engine calls for integer columns. It reads the input once, works out the encoded size of every candidate from what it saw (runs, distinct values, the dominant value, the min and width per 1024-value chunk), and only encodes with the winner. If the result ends up larger than Plain, it writes Plain instead. The id byte goes first, so `segment::decode_any` needs nothing else to read it back. Floats and strings go through `alp`, `alp_rd`, `fsst` and `zstd_leaf` directly. + +Every decoder takes a `max_values` (or `max_bytes`) ceiling from the caller and checks every count in the payload against it before allocating, so a corrupt or hostile file is an error and not a huge allocation. + +## Where it fits + +`zudb-zu1`, the native file format, is the main user. The ids and layouts are specified in `docs/04-storage-zu1-format.md`. The `zstd` feature links libzstd for compression; without it, zstd segments still decode through the pure Rust `ruzstd`, so the default build can read every file. + +## Example + +```rust +use zu_encoding::segment::{decode_any, encode_auto}; +use zu_encoding::{EncodingId, alp, fsst}; + +fn main() -> zu_common::Result<()> { + // Integers: the cascade prices every candidate and keeps the cheapest. + let ids: Vec = (1_000..3_048).collect(); + let mut seg = Vec::new(); + let chosen = encode_auto(&ids, &mut seg); + println!("{chosen:?}: {} values in {} bytes", ids.len(), seg.len()); + + let mut back = Vec::new(); + decode_any(&seg, ids.len(), &mut back)?; + assert_eq!(back, ids); + assert_eq!(EncodingId::try_from(seg[0])?, chosen); + + // Doubles that are really decimals, like prices. + let prices = [19.99, 5.25, 100.0, 0.5]; + let mut buf = Vec::new(); + alp::encode(&prices, &mut buf); + let mut out = Vec::new(); + alp::decode(&buf, prices.len(), &mut out)?; + assert_eq!(out, prices); + + // Strings: FSST replaces common substrings with one-byte codes. + let text = b"https://example.com/a https://example.com/b https://example.com/c"; + let mut packed = Vec::new(); + fsst::encode(text, &mut packed); + let mut plain = Vec::new(); + fsst::decode(&packed, text.len(), &mut plain)?; + assert_eq!(plain, text); + Ok(()) +} +``` + +On that input the cascade picks `DeltaBitPack` and stores 2048 ids in 287 bytes. + +## Should you depend on this? + +Probably not directly. This crate is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and all the zu crates are released together with exact version pins between them. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-encoding` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_encoding`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-exec/Cargo.toml b/crates/zu-exec/Cargo.toml index 8ac5d870..5733aef6 100644 --- a/crates/zu-exec/Cargo.toml +++ b/crates/zu-exec/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-exec/README.md b/crates/zu-exec/README.md new file mode 100644 index 00000000..da79d25d --- /dev/null +++ b/crates/zu-exec/README.md @@ -0,0 +1,74 @@ +# zudb-exec + +The push-based, morsel-parallel executor for zu. It runs the same `LogicalPlan`s as the executor in `zudb-query`, but it compiles them into a pipeline of push operators over column vectors and runs that pipeline on several threads at once. + +## How it works + +`try_execute` takes an optimized plan from `zudb-query` and goes through three steps. + +1. `compile` turns the plan into one pipeline: a scan at the bottom, then filters and expands, then a sink. If any part of the plan is a shape it does not handle, it returns `None` and nothing runs. +2. `run` splits the driving scan into morsels aligned to the storage's node groups. A shared counter hands them out in scan order to a persistent worker pool (`pool`). The calling thread is worker 0, and the others read through forked snapshot handles that share the block cache. Each worker pushes its morsel through the whole pipeline into its own sink, with no locks. +3. `sink` merges the per-worker results. Batches are put back together by morsel index, so a parallel run returns the same rows in the same order as a sequential one. LIMIT stops all the workers early once enough rows are in. + +All reads go through `zu_query::snapshot::Snapshot`, which hands out decoded columns a chunk at a time and pinned CSR slices, instead of the one-value-per-call `Graph` trait. + +The other modules: + +| Module | What it holds | +| --- | --- | +| `group` | the hash table behind GROUP BY and DISTINCT, with open addressing and keys packed into one flat buffer | +| `join` | the hash table behind hash joins, built once and shared by the workers instead of rebuilt by each one | +| `sip` | sideways information passing, where a join's build side tells the probe side which keys can match so it can skip the rest early | +| `decide` | the eight decisions the pipeline makes at runtime from what the data shows, each one counted and printed by EXPLAIN ANALYZE | +| `columns` | the sink for a plain projection, which keeps column vectors instead of building rows, so Arrow export can take them as they are | + +`PlanCache` holds the compiled pipeline of one statement so the next run with new parameters skips the compile step. The cache is checked against the snapshot epoch and the options before it is reused. + +## Using it + +Every entry point returns `Ok(None)` for a plan it does not cover, and the caller is expected to fall back to the row executor. This is the shape `zudb` uses: + +```rust +use zu_common::Result; +use zu_query::binder::{BoundQuery, Schema}; +use zu_query::exec::{self, Graph, Options, QueryResult, Value}; +use zu_query::plan::LogicalPlan; +use zu_query::snapshot::Snapshot; + +/// Try the pipeline first, and fall back to the row executor in +/// zu_query for any plan shape it does not cover yet. +fn run( + plan: &LogicalPlan, + query: &BoundQuery, + schema: &Schema, + snap: &mut dyn Snapshot, + graph: &mut dyn Graph, + params: &[Value], + options: &Options, +) -> Result { + if let Some(result) = zu_exec::try_execute(plan, query, schema, snap, params, options)? { + return Ok(result); + } + exec::execute(plan, query, schema, graph, params, options) +} +``` + +The other entry points have the same contract. `try_execute_cached` reuses a `PlanCache`, `try_execute_streaming` hands rows over in batches as they are made, and `try_execute_profiled` also returns the runtime decisions for EXPLAIN ANALYZE. With `options.flat` set they all return `None`, and `Options::engine` set to `Engine::Rows` makes `zudb` skip this crate entirely. + +You need a `Snapshot` implementation to call any of this. The real one reads a `zu1` file and lives in `zudb`. + +## What it covers today + +The linear read pipeline: one node scan, filters, single-hop expands (including a second pattern branch sharing a variable), and one final projection or aggregation with its DISTINCT, ORDER BY, SKIP, and LIMIT. Variable-length expands, bracketed groups, closing joins, UNWIND, table functions, and rel values still fall back to the row executor. Whatever this crate does run has to match the old executor exactly, including row order, group order, and errors on overflow, and the parity test suite in `zudb` holds it to that. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-exec` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_exec`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-json/Cargo.toml b/crates/zu-json/Cargo.toml index b78b483d..763ba41f 100644 --- a/crates/zu-json/Cargo.toml +++ b/crates/zu-json/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-json/README.md b/crates/zu-json/README.md new file mode 100644 index 00000000..ab3916cd --- /dev/null +++ b/crates/zu-json/README.md @@ -0,0 +1,51 @@ +# zudb-json + +A small JSON reader and writer, about 550 lines with no dependencies. It is used by the `zu` command-line tool for its line protocol and `--format json` output, and by the repository's codegen tool, which reads rustdoc's JSON output and writes the API model file. + +## Why it exists + +The `zu` binary has a 15 MiB size budget, and the JSON it handles is small: one-line protocol frames with strings, numbers, booleans and flat objects. A few hundred lines here cost less than a general JSON crate. The codegen tool needed the same thing at a larger size, a third of a megabyte of nested rustdoc output, and it needed the output file to be byte-for-byte identical from run to run so CI can diff it against the committed copy. That is why the reader is its own crate and why it keeps field order. + +If you need JSON in your own project, use `serde_json`. This crate does not do serde, streaming, or any of the conveniences you would expect from a general library. + +## Usage + +```rust +use zu_json::{Json, parse}; + +fn main() -> Result<(), String> { + let frame = parse(r#"{"op":"query","q":"MATCH (n) RETURN n","params":{"limit":10}}"#)?; + assert_eq!(frame.get("op").and_then(Json::as_str), Some("query")); + let limit = frame.get("params").and_then(|p| p.get("limit")).and_then(Json::as_i64); + assert_eq!(limit, Some(10)); + + let reply = Json::Obj(vec![ + ("gqlstatus".to_string(), Json::Str("00000".to_string())), + ("columns".to_string(), Json::Arr(vec![Json::Str("n".to_string())])), + ("rows".to_string(), Json::Arr(vec![])), + ]); + println!("{}", reply.to_compact()); + Ok(()) +} +``` + +## How it works + +Everything is in `lib.rs`. + +- `Json` is the value type: `Null`, `Bool`, `Int(i64)`, `Float(f64)`, `Str`, `Arr(Vec)` and `Obj(Vec<(String, Json)>)`. +- Objects are a vector of pairs, not a map. Field order survives a round trip, duplicate keys are kept rather than collapsed, and `get` is a linear scan, which is fast enough for the handful of fields a frame has. +- Accessors return `Option`: `get`, `as_str`, `as_bool`, `as_i64`, `as_u64`, `as_arr`, `as_obj`. An empty array and a missing field are different answers. +- `parse(text)` is a recursive descent parser. It reads exactly one value and rejects trailing input, refuses nesting deeper than 128 levels, and handles `\u` escapes including surrogate pairs. A number with no fraction or exponent that fits in an `i64` becomes `Int`, and anything else becomes `Float`. Errors are a `String` with the byte position. +- `to_compact()` writes one line with no spaces, and `to_pretty()` indents two spaces per level and ends with a newline. Floats are written as the shortest text that reads back to the same bits. NaN and infinity become `null` because JSON has no spelling for them. +- `escape_into(s, out)` escapes the quote, the backslash and the control characters, and leaves all other characters as UTF-8. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb-cli`](https://crates.io/crates/zudb-cli) can depend on it. Its API changes whenever zu needs it to, with no deprecation period, and the versions of all the zu crates move together. To use zu from Rust, add [`zudb`](https://crates.io/crates/zudb). + +The package is named `zudb-json` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_json`. + +## License + +Apache-2.0 diff --git a/crates/zu-query/Cargo.toml b/crates/zu-query/Cargo.toml index c3b2edd3..72a7b7a2 100644 --- a/crates/zu-query/Cargo.toml +++ b/crates/zu-query/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-query/README.md b/crates/zu-query/README.md new file mode 100644 index 00000000..5571240a --- /dev/null +++ b/crates/zu-query/README.md @@ -0,0 +1,140 @@ +# zudb-query + +The zu query engine. It takes the text of a zuQL statement (zu's dialect of ISO GQL) and turns it into rows: lexer, parser, binder, logical plan, optimizer, and a factorized vectorized executor. It knows nothing about files. It reads the graph through a small trait, so the same engine runs over all three of zu's storage engines. + +## How a statement runs + +``` +text -> lexer -> parser -> AST -> binder (+ Schema) -> BoundQuery + -> plan::build -> LogicalPlan -> optimizer -> exec::execute (+ Graph) -> QueryResult +``` + +- `lexer` and `parser` are hand-written. The parser is recursive descent over the grammar in `docs/grammar.ebnf` and covers MATCH, OPTIONAL MATCH, WHERE, CALL, UNWIND, WITH, and RETURN, plus the data-changing statements. Keywords match case-insensitively, errors name the line and column, and expression nesting is capped so hostile input cannot overflow the stack. +- `ast` is a plain description of the text. Anything that needs the catalog is left to the binder. +- `binder` resolves every variable to a slot, every label and rel type to a table, and types every expression. It binds against a `Schema`, a plain description of the node and rel tables plus whatever statistics the engine has, not against a storage engine. That is why it binds the same way over `zu1`, SQLite, and S3, and why its tests need no file. +- `plan` builds a left-deep `LogicalPlan`: scans introduce nodes, expands walk rels, predicates filter at the earliest point their variable exists. `plan::explain` renders it for EXPLAIN. +- `optimizer` orders joins by dynamic programming over the relationship subsets, driven by degree statistics when the engine has them, and places each filter at the earliest point it can run. +- `exec` splits the plan into stages at every projection or aggregation. Inside a stage, operators pull `Chunk`s of up to 2048 values. An expand keeps its output factorized, as a list of neighbors per source, until something needs one row at a time. Stages can run morsel-parallel when the graph handle can be forked. + +Around that core: + +| Module | What it holds | +| --- | --- | +| `functions`, `procedures` | the builtin function table and the procedure catalog the binder resolves names against | +| `cast`, `typed`, `value_type` | `CAST`, `IS TYPED`, and the type names that map onto the type lattice in `zudb-common` | +| `recursive` | the frontier BFS behind variable-length patterns, switching between sparse and dense frontiers at runtime | +| `csr`, `kernels` | an in-memory CSR and the graph kernels on it: BFS, shortest path, triangle count, PageRank, connected components | +| `column`, `row` | reading a result down its columns (what Arrow export uses) or as typed Rust tuples | +| `frame` | registering caller-owned columns, such as a dataframe, as a table a statement can read | +| `snapshot` | the vectorized read surface that the pipeline executor in `zudb-exec` uses | +| `refs` | graph and binding table reference values | + +## Using it + +The storage side is the `exec::Graph` trait. Only `neighbors`, `has_edge`, and `property` have to be written; the rest have defaults. This runs a two-hop chain held in a struct, which is about the smallest graph the engine will accept. `Result` comes from `zudb-common`, so add that crate as well if you try it. + +```rust +use zu_common::Result; +use zu_query::binder::{self, NodeDef, RelDef, Schema}; +use zu_query::exec::{self, Graph, Options, Value}; +use zu_query::{optimizer, parser, plan}; + +/// Three people in a chain: 0 knows 1, 1 knows 2. +struct Chain; + +impl Graph for Chain { + fn neighbors( + &mut self, + _rel: u32, + node: u64, + reversed: bool, + out: &mut Vec, + ) -> Result<()> { + out.clear(); + let next = if reversed { + node.checked_sub(1) + } else { + Some(node + 1).filter(|&n| n < 3) + }; + out.extend(next); + Ok(()) + } + + fn has_edge(&mut self, _rel: u32, src: u64, dst: u64) -> Result { + Ok(dst == src + 1 && dst < 3) + } + + fn property(&mut self, _table: u32, offset: u64, key: &str) -> Result { + Ok(match key { + "name" => Value::Str(["ada", "grace", "edsger"][offset as usize].into()), + _ => Value::Null, + }) + } +} + +fn main() -> Result<()> { + let schema = Schema::new( + vec![NodeDef { + id: 0, + name: "person".into(), + node_count: 3, + labels: Vec::new(), + }], + vec![RelDef { + id: 1, + name: "knows".into(), + from: 0, + to: 0, + edge_count: 2, + undirected: false, + }], + )?; + + let parsed = parser::parse("MATCH (a:person)-[:knows]->(b:person) RETURN a.name, b.name")?; + let query = binder::bind(&parsed, &schema)?; + let optimized = optimizer::optimize(plan::build(&query)?, &query, &schema)?; + println!("{}", plan::explain(&optimized, &query, &schema)); + + let result = exec::execute( + &optimized, + &query, + &schema, + &mut Chain, + &[], + &Options::default(), + )?; + for row in result.rows.iter() { + println!("{row:?}"); + } + Ok(()) +} +``` + +That prints the plan and then two rows: + +``` +Project a.name, b.name + Expand (a)-[#1:knows]->(b) + ScanNodes a: person + +[Str("ada"), Str("grace")] +[Str("grace"), Str("edsger")] +``` + +This is the same sequence `zudb` runs for every read. In `zudb` the `Schema` comes from the file's catalog and the `Graph` is a reader over the file, and parameters, transactions, and caching of compiled plans are all handled there. + +## Two executors + +The executor in this crate is the older, pull-based one. Plans that `zudb-exec` covers run there, and this one stays as the fallback and as the reference the new executor is tested against. The full plan for the language and the operators is in `docs/07-query-engine.md`. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-query` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_query`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-s3/Cargo.toml b/crates/zu-s3/Cargo.toml index c69b8cec..6288de1c 100644 --- a/crates/zu-s3/Cargo.toml +++ b/crates/zu-s3/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-s3/README.md b/crates/zu-s3/README.md new file mode 100644 index 00000000..7d521fd5 --- /dev/null +++ b/crates/zu-s3/README.md @@ -0,0 +1,73 @@ +# zudb-s3 + +The `s3` storage engine for zu: a graph that lives in object storage (S3, GCS, Azure Blob, R2, MinIO and anything else `object_store` supports), with a fixed number of requests per second no matter how busy the writers are. + +Most of this engine is not built yet. This README describes the design and then says exactly which part exists today. + +## The design + +The full protocol is in [`docs/06-storage-s3.md`](https://github.com/tamnd/zu/blob/main/docs/06-storage-s3.md). The idea is to borrow what SlateDB, WarpStream and Quickwit learned about object storage: + +- Data lives in immutable segment packs: many `zu1` format segments in one 8 to 64 MiB object, with a footer that indexes them, so a cold read of any segment is one ranged GET. +- Writes go to a WAL buffer that is flushed as one object every 100 ms or every 4 MiB, whichever comes first. That caps PUTs at about 10 per second however many transactions there are, which is what keeps the monthly bill flat. +- A manifest object is the root of each database state. A commit writes a new manifest and swings a pointer to it with a conditional PUT, so a half-finished commit is never visible. +- There is one writer at a time, enforced by epochs rather than leases or a lock service. A new writer takes over by committing the next epoch. After that, every conditional PUT from the old writer fails, so it can't overwrite anything. +- Reads go through a RAM and local NVMe cache, with the adjacency and footers pinned so a traversal does not go to S3 one node at a time. + +## What exists today + +The manifest format and the commit protocol with epoch fencing. That is the piece everything else depends on, because it decides who is allowed to write and what the current state is. + +- `Manifest` is one snapshot: an `epoch`, the `writer_id` allowed to produce the next epoch, and the object keys of its segment packs. It encodes to a small versioned binary format (magic `ZUS3`, little endian, CRC32C at the end). Decoding never panics: truncated or corrupt bytes give an error. +- `ManifestStore` reads and commits manifests for one writer. Every commit first writes an immutable snapshot at `manifest/{epoch}.zum`, then replaces `manifest/CURRENT` with a conditional PUT. `CURRENT` holds the whole manifest rather than a pointer, so opening a database is one GET. +- `commit` requires the epoch to go up by exactly one and the writer to be the one the current manifest names. Losing a race, or being fenced by a newer writer, returns a `Conflict` error. +- `take_over` makes this store the writer: it republishes the current segment set under the next epoch with its own `writer_id`. + +The crate has no async runtime of its own. It drives `object_store`'s futures on the calling thread, so you can call it from synchronous code. + +Not built yet: the WAL objects and their flusher, segment packs, the checkpoint that folds the WAL into packs, garbage collection, and the cache. `zudb::Database` cannot open an S3 database yet. + +## Usage + +You bring an `object_store` backend. This example uses the in-memory one; for S3 you would build an `AmazonS3` store the same way and pass it in. + +```rust +use std::sync::Arc; + +use object_store::memory::InMemory; +use zu_s3::{Manifest, ManifestStore}; + +fn main() -> Result<(), Box> { + let bucket = Arc::new(InMemory::new()); + + // The first writer creates the database at epoch 0. + let a = ManifestStore::new(bucket.clone(), 1); + let first = Manifest { epoch: 0, writer_id: 1, segments: vec![] }; + let mut current = a.commit(&first, None)?; + + // Each commit advances the epoch by one. + let next = Manifest { epoch: 1, writer_id: 1, segments: vec!["seg/0001.zuseg".into()] }; + current = a.commit(&next, Some(¤t))?; + + // A second writer takes over, and from then on the first one is fenced. + let b = ManifestStore::new(bucket.clone(), 2); + let taken = b.take_over()?; + println!("writer {:#x} owns epoch {}", taken.manifest.writer_id, taken.manifest.epoch); + + let stale = Manifest { epoch: 2, writer_id: 1, segments: vec![] }; + assert!(a.commit(&stale, Some(¤t)).is_err()); + Ok(()) +} +``` + +The backend has to support conditional PUTs. S3 (since late 2024), GCS, Azure and R2 all do. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-s3` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_s3`. + +## License + +Apache-2.0 diff --git a/crates/zu-snippets/examples/bulk-and-stream.rs b/crates/zu-snippets/examples/bulk-and-stream.rs new file mode 100644 index 00000000..23285dc0 --- /dev/null +++ b/crates/zu-snippets/examples/bulk-and-stream.rs @@ -0,0 +1,27 @@ +use zudb::{Database, Flow}; + +fn main() -> zudb::Result<()> { + let db = Database::memory()?; + let mut conn = db.connect()?; + conn.execute("INSERT (p:person {uid: 0, name: 'first'})")?; + + let mut app = conn.appender("person")?; + for uid in 1..=10_000i64 { + app.append_row((uid, "someone"))?; + } + let loaded = app.close()?; + println!("appended {loaded}"); + + let mut total = 0i64; + conn.query_stream("MATCH (p:person) RETURN p.uid AS uid", &[], |batch| { + for row in batch.iter() { + total += row.get_at::(0)?; + } + Ok(Flow::More) + })?; + println!("sum {total}"); + + let plan = conn.explain("MATCH (p:person) WHERE p.uid > 9000 RETURN p.name AS name")?; + println!("{plan}"); + Ok(()) +} diff --git a/crates/zu-snippets/examples/transactions.rs b/crates/zu-snippets/examples/transactions.rs new file mode 100644 index 00000000..f0d4c9c5 --- /dev/null +++ b/crates/zu-snippets/examples/transactions.rs @@ -0,0 +1,31 @@ +use zudb::{Database, params}; + +fn main() -> zudb::Result<()> { + let db = Database::memory()?; + let mut conn = db.connect()?; + + conn.execute("START TRANSACTION")?; + conn.execute("INSERT (a:person {uid: 1, name: 'ada'})-[:knows {since: 1843}]->(b:person {uid: 2, name: 'charles'})")?; + conn.execute("COMMIT")?; + + conn.execute("START TRANSACTION")?; + conn.execute("INSERT (p:person {uid: 3, name: 'nobody'})")?; + conn.execute("ROLLBACK")?; + + let (stmt, names) = conn.prepare( + "MATCH (a:person)-[k:knows]->(b:person) WHERE a.uid = $uid RETURN b.name AS name, k.since AS since", + )?; + assert_eq!(names, ["uid"]); + let rows = conn.execute_prepared(stmt, ¶ms! { "uid" => 1 })?; + for row in rows.iter() { + let name: &str = row.get_by_name("name")?; + let since: i64 = row.get_by_name("since")?; + println!("ada knows {name} since {since}"); + } + conn.close_prepared(stmt); + + let people = conn.query("MATCH (p:person) RETURN count(p) AS n")?; + let n: i64 = people.row(0)?.get_at(0)?; + println!("{n} people"); + Ok(()) +} diff --git a/crates/zu-snippets/src/lib.rs b/crates/zu-snippets/src/lib.rs index 72fb305d..02ae7766 100644 --- a/crates/zu-snippets/src/lib.rs +++ b/crates/zu-snippets/src/lib.rs @@ -23,3 +23,8 @@ /// The examples this package holds, in the order the README prints /// them, and the name of the fenced block each one has to match. pub const SNIPPETS: &[&str] = &["sixty-seconds"]; + +/// The examples the `zudb` README prints, which is the page crates.io +/// shows. It opens with the same quickstart and then goes further, +/// so it has a list of its own. +pub const ZUDB_SNIPPETS: &[&str] = &["sixty-seconds", "transactions", "bulk-and-stream"]; diff --git a/crates/zu-snippets/tests/readme.rs b/crates/zu-snippets/tests/readme.rs index e66d3776..b1bda4c5 100644 --- a/crates/zu-snippets/tests/readme.rs +++ b/crates/zu-snippets/tests/readme.rs @@ -69,28 +69,39 @@ fn example(name: &str) -> PathBuf { built } +/// The READMEs that print the snippets, each with the examples it +/// prints in order: the repository's, and the one crates.io shows on +/// the `zudb` page, which starts with the same quickstart and would +/// drift the same way. +const READMES: [(&str, &[&str]); 2] = [ + ("README.md", zu_snippets::SNIPPETS), + ("crates/zu/README.md", zu_snippets::ZUDB_SNIPPETS), +]; + #[test] fn the_readme_prints_the_program_this_repository_compiles() { - let readme = std::fs::read_to_string(root().join("README.md")).expect("a README"); - let printed = blocks(&readme, "rust"); - assert_eq!( - printed.len(), - zu_snippets::SNIPPETS.len(), - "the README prints {} Rust blocks and this package holds {} snippets", - printed.len(), - zu_snippets::SNIPPETS.len() - ); - for (block, name) in printed.iter().zip(zu_snippets::SNIPPETS) { - let path = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("examples") - .join(format!("{name}.rs")); - let source = std::fs::read_to_string(&path).expect("the example is there"); + for (file, snippets) in READMES { + let readme = std::fs::read_to_string(root().join(file)).expect("a README"); + let printed = blocks(&readme, "rust"); assert_eq!( - block, - &source, - "the README block and {} have drifted apart", - path.display() + printed.len(), + snippets.len(), + "{file} prints {} Rust blocks and lists {} snippets", + printed.len(), + snippets.len() ); + for (block, name) in printed.iter().zip(snippets) { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("examples") + .join(format!("{name}.rs")); + let source = std::fs::read_to_string(&path).expect("the example is there"); + assert_eq!( + block, + &source, + "the block in {file} and {} have drifted apart", + path.display() + ); + } } } @@ -111,3 +122,36 @@ fn the_sixty_second_program_runs_and_prints_what_the_readme_says() { // which is the part of the story a compile check cannot see. assert!(dir.path().join("social.zu1").is_file()); } + +/// Runs an example in a directory of its own and hands back what it +/// printed, failing the test if it did not exit cleanly. +fn run(name: &str) -> String { + let dir = tempfile::tempdir().expect("a directory of its own"); + let run = Command::new(example(name)) + .current_dir(dir.path()) + .output() + .expect("the example runs"); + assert!( + run.status.success(), + "{name} failed: {}", + String::from_utf8_lossy(&run.stderr) + ); + String::from_utf8_lossy(&run.stdout).into_owned() +} + +#[test] +fn the_transactions_program_keeps_the_commit_and_drops_the_rollback() { + assert_eq!( + run("transactions"), + "ada knows charles since 1843\n2 people\n" + ); +} + +#[test] +fn the_bulk_program_loads_streams_and_explains() { + assert_eq!( + run("bulk-and-stream"), + "appended 10000\nsum 50005000\n\ + Project p.name AS name\n Filter p.uid > 9000\n ScanNodes p: person\n\n" + ); +} diff --git a/crates/zu-sqlite/Cargo.toml b/crates/zu-sqlite/Cargo.toml index ed07d449..4c268c40 100644 --- a/crates/zu-sqlite/Cargo.toml +++ b/crates/zu-sqlite/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-sqlite/README.md b/crates/zu-sqlite/README.md new file mode 100644 index 00000000..8aa87ba6 --- /dev/null +++ b/crates/zu-sqlite/README.md @@ -0,0 +1,65 @@ +# zudb-sqlite + +The `sqlite` storage engine for zu: a graph stored in an ordinary SQLite database file. + +There are two reasons it exists. A SQLite file can be opened, inspected and backed up with tools people already have, which matters where a custom format is a hard sell. And because it is simple and SQLite is well tested, it is the reference zu's native `zu1` engine is checked against: the same queries run on both, and the results have to match. + +## How the graph maps onto SQLite + +The mapping is specified in [`docs/05-storage-sqlite.md`](https://github.com/tamnd/zu/blob/main/docs/05-storage-sqlite.md). In this crate today: + +- A node table `person` becomes `n_person (zrow INTEGER PRIMARY KEY, p_name ..., ...)`. `zrow` is the node id and each property column gets a `p_` prefix. +- A rel table `follows` becomes `r_follows (zrel INTEGER PRIMARY KEY, src, dst, p_...)` with two indexes, `(src, dst)` and `(dst, src)`, so neighbors in either direction are an index range scan. +- `zu_catalog` lists every table with a stable id, its kind, its DDL, the endpoint node tables of a rel table, and whether the edges are undirected. `zu_labels` holds any labels a node carries beyond its table name. +- The file is tagged with `application_id = 0x5A5531` (the bytes `ZU1`) and `user_version` is the schema version, currently 4. Older files are migrated on open. A file that belongs to some other application is rejected. +- Opening applies a fixed pragma profile: WAL journal, `synchronous=NORMAL`, 8 KiB pages, a 16 MiB page cache, no mmap, and a 5 second busy timeout. +- Types map onto SQLite's storage classes. Booleans, dates, times and durations are stored as integers, and the declared column type (`ColumnType`) records which one a column holds. List columns are stored as JSON text. + +For traversals, `SqliteStore::csr` builds the adjacency of one node group (131,072 rows) in CSR form from the index and caches it. Each write bumps the version of only the groups it touches, so the cache is invalidated per group rather than per table, and a traversal over unchanged data runs off the cache instead of going back to SQLite. + +Transactions are single-writer: `begin` is `BEGIN IMMEDIATE`, and `checkpoint` runs `wal_checkpoint(TRUNCATE)`. `epoch` gives a commit counter that moves on every write from any connection. + +## Usage + +`SqliteStore` is the whole API: open a file, create tables, write rows inside a transaction, and read properties and neighbors back. + +```rust +use zu_sqlite::{ColumnType, Direction, SqliteStore, Value}; + +fn main() -> Result<(), Box> { + let mut store = SqliteStore::open("social.db")?; + store.create_node_table("person", &[("name", ColumnType::Text)])?; + store.create_rel_table("follows", "person", "person", &[])?; + + store.begin()?; + let ada = store.insert_node("person", &[Value::Text("ada".into())])?; + let grace = store.insert_node("person", &[Value::Text("grace".into())])?; + store.insert_rel("follows", ada, grace, &[])?; + store.commit()?; + + let name = store.read_node_prop("person", grace, "name")?; + let follows = store.neighbors("follows", ada, Direction::Fwd)?; + println!("ada follows {follows:?}, which is {name:?}"); + + store.checkpoint()?; + Ok(()) +} +``` + +This crate has no query language. To run zuQL against a SQLite file, `zudb` has `zudb::sqlite::run(statement, &store, params)`, which uses the same parser, optimizer and executor as the `zu1` engine. To move a graph between the two engines, use `zudb::convert::zu1_to_sqlite` and `zudb::convert::sqlite_to_zu1`, or `zu convert graph.zu1 graph.db` from the CLI. + +SQLite is bundled through `rusqlite`, so nothing needs to be installed on the system. + +## Status + +`zudb::Database` opens `zu1` files only, so the SQLite engine is reached through `zudb::sqlite` and the conversion functions rather than through `Database`. It does not yet implement the `GraphStore` trait from `zudb-storage`. That waits on the shared buffer manager described in `docs/09`. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-sqlite` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_sqlite`. + +## License + +Apache-2.0 diff --git a/crates/zu-storage/Cargo.toml b/crates/zu-storage/Cargo.toml index d94759de..0efdd076 100644 --- a/crates/zu-storage/Cargo.toml +++ b/crates/zu-storage/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-storage/README.md b/crates/zu-storage/README.md new file mode 100644 index 00000000..0d5b5531 --- /dev/null +++ b/crates/zu-storage/README.md @@ -0,0 +1,86 @@ +# zudb-storage + +The contract between the [zu](https://github.com/tamnd/zu) query engine and its storage engines. + +zu has one query processor and three storage engines: the native single-file `zu1` format, an ordinary SQLite file, and object storage (`s3`). This crate is where the trait they are meant to share lives, so the query layer can be written once against it. + +## What is in it + +- `GraphStore` is a storage engine. It hands out the `Catalog`, takes a consistent `Snapshot` for reading, applies a `CommitBatch` and returns the new `Epoch`, runs a checkpoint, and ingests sealed node groups from a bulk load. +- `Snapshot` is a read view at one epoch. It returns a column segment for a node group (`scan_column`), the CSR adjacency for a node group in one `Direction` (`csr`), and looks up a node by primary key (`lookup_pk`). +- `Direction` (`Fwd` or `Bwd`) and `CheckpointMode` (`Normal` or `Full`). + +Both traits are `Send + Sync`, and snapshots are handed out as `Arc`, so one store can serve readers on many threads while a single writer commits. + +## Status + +This crate is mostly a skeleton today, and it is worth being plain about that. `Catalog`, `CommitBatch`, `SealedNodeGroup`, `SegmentRef` and `CsrRef` are empty structs, and none of the three engines implements `GraphStore` yet. Each engine currently exposes its own API, and the `zudb` crate wires them to the query engine directly. What is used for real right now is `Direction`. The full trait surface is specified in `docs/02-architecture.md`, and this crate will grow into it as the engines converge on a shared buffer manager. + +## Example + +The trait is small enough to show whole. This is a store with nothing in it: + +```rust +use std::sync::Arc; + +use zu_common::{Epoch, NodeGroupId, NodeOffset, Result, TableId}; +use zu_storage::{ + Catalog, CheckpointMode, CommitBatch, CsrRef, Direction, GraphStore, SealedNodeGroup, + SegmentRef, Snapshot, +}; + +/// A store with nothing in it, to show the shape of the contract. +struct Empty; + +impl Snapshot for Empty { + fn scan_column(&self, _t: TableId, _g: NodeGroupId, _c: u32) -> Result { + Ok(SegmentRef {}) + } + fn csr(&self, _t: TableId, _g: NodeGroupId, _dir: Direction) -> Result { + Ok(CsrRef {}) + } + fn lookup_pk(&self, _t: TableId, _key: &[u8]) -> Result> { + Ok(None) + } + fn epoch(&self) -> Epoch { + 0 + } +} + +impl GraphStore for Empty { + fn catalog(&self) -> Arc { + Arc::new(Catalog::default()) + } + fn snapshot(&self) -> Arc { + Arc::new(Empty) + } + fn commit(&self, _batch: CommitBatch) -> Result { + Ok(1) + } + fn checkpoint(&self, _mode: CheckpointMode) -> Result<()> { + Ok(()) + } + fn ingest(&self, _groups: Vec) -> Result { + Ok(1) + } +} + +fn main() -> Result<()> { + let store: Arc = Arc::new(Empty); + let snap = store.snapshot(); + assert_eq!(snap.lookup_pk(0, b"ada")?, None); + store.checkpoint(CheckpointMode::Normal) +} +``` + +## Should you depend on this? + +Probably not directly. This crate is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and all the zu crates are released together with exact version pins between them. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-storage` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_storage`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-vector/Cargo.toml b/crates/zu-vector/Cargo.toml index 7f4efc95..8edb689d 100644 --- a/crates/zu-vector/Cargo.toml +++ b/crates/zu-vector/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-vector/README.md b/crates/zu-vector/README.md new file mode 100644 index 00000000..6066d87f --- /dev/null +++ b/crates/zu-vector/README.md @@ -0,0 +1,86 @@ +# zudb-vector + +Typed columnar vectors and the expression kernels that run over them, for the [zu](https://github.com/tamnd/zu) query executor. + +Instead of a row of tagged values per cell, a column is one contiguous buffer of one physical type, nulls are a separate bitmap, and a filter produces a list of row indices instead of copying rows. Kernels are plain Rust loops over those buffers, written so the compiler can auto-vectorize them. + +## The pieces + +| Type | What it is | +| --- | --- | +| `MorselArena` | A bump allocator in 256 KiB blocks. Every buffer for one morsel of work comes from it, and `reset` keeps the blocks, so a warm executor allocates nothing per morsel. | +| `ValueVector` | One column: a `PhysType`, an encoding (`Flat`, `Constant`, or `Dict` codes), the data buffer, and an optional validity `Bitmap`. | +| `PhysType` | `Bool`, `Int64`, `Float64`, `NodeRef`, `RelRef`, `Str`, `List`, `Interval`, `Date`, `Timestamp`. One word per value (bools are bit-packed, strings are 16 bytes). | +| `Bitmap` | One bit per row, used for validity and for predicate results. Null handling is a word-wise AND. | +| `SelVector` | A sorted list of `u16` row indices. Filters refine a selection and never move data. | +| `StrView`, `StrBuffers`, `Dictionary` | Strings as 16-byte views: up to 12 bytes inline, otherwise a 4-byte prefix plus a buffer id and offset. Dictionary-encoded strings stay as integer codes until the result is produced. | +| `DataChunk`, `ChunkSet` | A chunk is a set of vectors with one shared selection, usually `VECTOR_SIZE` (2048) rows. A chunk set is a factorized result: a flat chunk plus list-level chunks whose sizes multiply. | +| `Program`, `ExprOp` | An expression compiled into a short register program. Evaluating it walks the ops, not an expression tree. | +| `kernels` | Comparison, arithmetic, math, string, temporal, hashing, gather, set and aggregate kernels. | + +## How data flows + +The executor in `zudb-exec` splits a scan into morsels. For each morsel it takes an arena, reads columns from storage into `ValueVector`s, and builds a `DataChunk`. A WHERE clause is a `Program` that ends in a comparison, so `eval_filter` returns a `Bitmap`, which becomes a `SelVector` on the chunk. Projections are programs evaluated with `eval`, which return a new vector. When the morsel is done the results are copied into the sink and the arena is reset for the next one. + +Buffers from the arena are not tied to it by a lifetime. The rule is that a vector never outlives its morsel unless it is copied out, and debug builds check that on every access. + +## Where it fits + +It depends only on `zudb-common`. `zudb-exec` is built on it, and `zudb-query` and `zudb` use it for the vectorized snapshot reads. + +## Example + +```rust +use zu_vector::{ + BinOp, CmpOp, DataChunk, ExprOp, MorselArena, OwnedValue, PhysType, Program, SelVector, + ValueVector, +}; + +fn main() -> zu_common::Result<()> { + // Every buffer for one morsel comes from this arena. + let mut arena = MorselArena::new(); + let ages = ValueVector::flat_from(&mut arena, PhysType::Int64, &[17i64, 34, 52, 8]); + let chunk = DataChunk::new(vec![ages], 4); + + // WHERE age >= 18, compiled once into a register program. + let filter = Program { + ops: vec![ + ExprOp::LoadCol { col: 0, dst: 0 }, + ExprOp::LoadConst { v: OwnedValue::Int(18), dst: 1 }, + ExprOp::Compare { op: CmpOp::Ge, l: 0, r: 1, dst: 2 }, + ], + regs: 3, + }; + let bits = filter.eval_filter(&chunk, &mut arena)?; + let sel = SelVector::from_bitmap(&mut arena, &bits); + assert_eq!(sel.as_slice(), &[1, 2]); + + // RETURN age * 12, evaluated a whole vector at a time. + let months = Program { + ops: vec![ + ExprOp::LoadCol { col: 0, dst: 0 }, + ExprOp::LoadConst { v: OwnedValue::Int(12), dst: 1 }, + ExprOp::Binary { op: BinOp::Mul, l: 0, r: 1, dst: 2 }, + ], + regs: 3, + }; + let out = months.eval(&chunk, &mut arena)?; + assert_eq!(out.values::(), &[204, 408, 624, 96]); + + // The next morsel reuses the same memory. + arena.reset(); + Ok(()) +} +``` + +## Should you depend on this? + +Probably not directly. This crate is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and all the zu crates are released together with exact version pins between them. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-vector` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_vector`. + +zu is early and not ready for real data yet. See the [repository](https://github.com/tamnd/zu) for the status and the specification. + +## License + +Apache-2.0 diff --git a/crates/zu-zu1/Cargo.toml b/crates/zu-zu1/Cargo.toml index b062321e..e524285a 100644 --- a/crates/zu-zu1/Cargo.toml +++ b/crates/zu-zu1/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +readme = "README.md" # Published as zudb-*, because zu is taken on crates.io, but the # library keeps the name every crate in this workspace already uses. diff --git a/crates/zu-zu1/README.md b/crates/zu-zu1/README.md new file mode 100644 index 00000000..182adab7 --- /dev/null +++ b/crates/zu-zu1/README.md @@ -0,0 +1,82 @@ +# zudb-zu1 + +The `zu1` storage engine: zu's native format, where one `.zu1` file holds the whole graph. + +When you call `zudb::Database::create("social.zu1")`, this is the crate that writes the file. It owns the bytes on disk: headers, blocks, column segments, adjacency, the catalog, and the write-ahead log next to the file. The query engine in `zudb-query` reads through it, and `zudb` puts the public API on top. + +## How a file is laid out + +The format is specified byte by byte in [`docs/04-storage-zu1-format.md`](https://github.com/tamnd/zu/blob/main/docs/04-storage-zu1-format.md). The short version: + +- Block 0 holds a write-once 4 KiB file header (magic `図ZU1`, format version, block size) and two 4 KiB database header slots. Everything after that is fixed 32 KiB blocks. +- Opening reads those 12 KiB and picks the valid database header with the highest epoch. New data always goes to free blocks, and a checkpoint publishes it by writing the other header slot. A torn header write leaves the previous one intact, so committed state is never damaged. +- The database header points at a few roots: the catalog (node and rel tables), the table index (where each table's data lives), statistics, and the free list. Each root is a meta chain, a linked list of checksummed blocks. +- Rows live in node groups of 131,072. Each property column is a segment. Fixed-width values use the MiniBlock layout: chunks of 1024 values, each compressed on its own with the cascades from `zudb-encoding`, so a point read only decodes the chunk it needs. Variable-width values (strings, blobs, lists) use FullZip, which keeps any row range contiguous and makes seeking to a row cheap. Byte rows of equal length use Stride. +- Edges are stored as CSR per node group, once keyed by source and once keyed by destination, so out-neighbors and in-neighbors are both direct reads. + +## Modules + +| Module | What it does | +| --- | --- | +| `file` | `Zu1File`: headers, block I/O, the block cache hookup, the dual-header checkpoint, savepoints | +| `meta` | Meta chains behind every root pointer | +| `catalog` | Table definitions and the table index | +| `segment`, `fullzip`, `rows` | The MiniBlock, FullZip and Stride column layouts | +| `graph` | Bulk-loaded CSR adjacency, edge list readers, `GraphReader` | +| `props` | Node and edge property columns with their logical types | +| `keys`, `reorder` | Primary-key index, and relabeling node ids by degree or BFS before a load so neighbors get close ids | +| `wal`, `txn`, `epoch` | Redo-only WAL, the single-writer commit path with MVCC overlays, and the snapshot epochs readers pin | +| `fold` | The checkpoint that folds overlays into new sealed segments and publishes them with one header flip | +| `ingest` | Bulk loads that write segments directly and log only a reference to them | +| `cache`, `vfs` | The shared block cache, and the file shim that the crash harness and in-memory databases swap in for the real filesystem | +| `stats`, `colors` | Degree histograms and color summaries the optimizer uses for cardinality estimates | +| `algo` | Whole-graph kernels over the CSR: BFS, PageRank, WCC, SSSP, CDLP, LCC, triangle count, Louvain | +| `parquet` | Parquet edge lists, behind the `arrow` feature | + +The crate also exports `verify(path)`, which walks every checksum in a file and cross-checks the structures against each other, and `layout(path)`, which tells you how much of a file is schema, free space, and graph. These are what `zu verify` and `zu stat` print. + +## Usage + +Most code should use `zudb`, which runs queries against a zu1 file and handles transactions for you. Using this crate directly is for tools that work with the file itself, like bulk loaders and checkers. Here is a bulk load of an edge list and a neighbor read: + +```rust +use std::path::Path; + +use zu_zu1::file::Zu1File; +use zu_zu1::graph::{GraphReader, bulk_load_as}; + +fn main() -> Result<(), Box> { + let path = Path::new("follows.zu1"); + + // Five people, and who follows whom, as dense row ids. + let mut edges = vec![(0, 1), (0, 2), (1, 2), (3, 0), (4, 3)]; + edges.sort_unstable(); + + let mut db = Zu1File::create(path)?; + bulk_load_as(&mut db, "person", "follows", 5, &edges)?; + + let mut reader = GraphReader::load_table(&mut db, "follows")?; + println!("0 follows {:?}", reader.neighbors(&mut db, 0)?); + + let bytes = zu_zu1::verify(path)?; + let layout = zu_zu1::layout(path)?; + println!("verified {bytes} bytes, {} blocks of graph", layout.data_blocks); + Ok(()) +} +``` + +`zudb` re-exports this crate as `zudb::zu1`, so if you already depend on `zudb` you can reach these types without adding this crate. + +## Status + +This is the most complete of the three engines and the one `zudb::Database` uses. The format version is 1. Until zu reaches 1.0 the format can still change between releases, and `verify` is the tool to run if you suspect a file. + +## Should you depend on this? + +Probably not directly. This crate is one part of [zu](https://github.com/tamnd/zu), an embedded property-graph database, and it is published so that [`zudb`](https://crates.io/crates/zudb) can depend on it. Its API changes whenever the engine needs it to, with no deprecation period, and the versions of all the zu crates move together. If you want to use zu from Rust, add `zudb` and use what it exports. + +The package is named `zudb-zu1` because `zu` was already taken on crates.io. The library keeps the name the workspace uses, so the import is `use zu_zu1`. + +## License + +Apache-2.0 diff --git a/crates/zu/Cargo.toml b/crates/zu/Cargo.toml index 646efffb..2b40e8fd 100644 --- a/crates/zu/Cargo.toml +++ b/crates/zu/Cargo.toml @@ -7,7 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true -readme = "../../README.md" +readme = "README.md" keywords = ["database", "graph", "embedded", "gql", "columnar"] categories = ["database-implementations"] diff --git a/crates/zu/README.md b/crates/zu/README.md new file mode 100644 index 00000000..b4f6e289 --- /dev/null +++ b/crates/zu/README.md @@ -0,0 +1,175 @@ +# zudb + +zu is an embedded, in-process property-graph database written in Rust. There is no server: you link the crate, point it at a file, and run graph queries in your own process. Storage is columnar and compressed, the executor is vectorized and factorized, and queries are written in zuQL, a dialect of ISO GQL. + +This crate is the one to depend on. It is published as `zudb` because the name `zu` was already taken on crates.io. The repository, the `zu` binary, and the `.zu1` file extension keep the short name. + +``` +cargo add zudb +``` + +zu is early. The specification in [docs/](https://github.com/tamnd/zu/tree/main/docs) is complete and the engine behind it is not, so this is a crate for trying the API and not yet one for keeping data you care about. Expect breaking changes between 0.0.x releases. + +## Quickstart + +```rust +use zudb::{Database, params}; + +fn main() -> zudb::Result<()> { + let db = Database::create("social.zu1")?; + let mut conn = db.connect()?; + + conn.execute("INSERT (p:person {uid: 1, name: 'ada'})")?; + conn.execute("INSERT (p:person {uid: 2, name: 'grace'})")?; + + let rows = conn.query_with( + "MATCH (p:person) WHERE p.uid >= $uid RETURN p.name AS name, p.uid AS uid", + ¶ms! { "uid" => 1 }, + )?; + for row in rows.iter() { + let (name, uid): (&str, i64) = row.get()?; + println!("{name} {uid}"); + } + Ok(()) +} +``` + +There is no schema step. The first `INSERT` that names a label creates its table, with the properties it was given as the columns. `create` makes a new file and fails if one is already there, so a program that wants the file it made last time calls `open`. + +## Databases and connections + +A `Database` is a path and a configuration, checked against a real file when you open it. It holds no file handle and no cache, so it is cheap to clone and safe to share between threads. A `Connection` is where the state lives: a file handle, decoded column caches, and a cache of compiled plans. Take one connection per thread with `db.connect()`, or `conn.duplicate()` to get another one off an existing connection, which is how a pool is written. + +| Call | What it does | +| --- | --- | +| `Database::create(path)` | Makes a new empty `.zu1` file. Fails if the path exists. | +| `Database::open(path)` | Opens an existing file. Reads 12 KiB up front and pages the rest in lazily. | +| `Database::memory()` | A database that never touches the filesystem. Same code paths, gone when the last handle is dropped. | +| `Database::open_with(path, Config)` | Open with settings: `memory_limit`, `threads`, `read_only`, `stale_bound`. | + +A connection runs one statement at a time. Every method takes `&mut self` and the type is `Send` but not `Sync`, so the compiler stops two threads from sharing one. Each statement reads a snapshot taken when it starts, and a commit made on another connection is visible to the next statement without reconnecting. Writes are serialized per file: one connection holds the write side for the length of a write statement or an explicit transaction, and the others wait their turn. + +## Running statements + +- `conn.query(text)` runs a statement and returns a `QueryResult` with `columns` and `rows`. +- `conn.query_with(text, ¶ms!{ "name" => value })` binds parameters by name. The names have no `$`. +- `conn.execute(text)` runs a statement for its effect. +- `conn.prepare(text)` compiles once and returns an id and the parameter names, then `conn.execute_prepared(id, ¶ms)` runs it as often as you like and `conn.close_prepared(id)` releases it. +- `conn.query_stream(text, ¶ms, |batch| ...)` hands rows to a closure in batches instead of collecting them. Return `Flow::Stop` to end the scan early. +- `conn.explain(text)` prints the plan without running it, `conn.explain_plan(text)` returns it as a tree, and `conn.profile(text, ¶ms)` runs it with per-operator counters. +- `conn.interrupt()` gives you a handle another thread can use to stop the running statement, which then fails with `ZuError::Interrupted`. + +Rows are read as Rust types. `row.get::<(&str, i64)>()` reads the whole row as a tuple, `row.get_at::(0)` reads one column by position and `row.get_by_name::<&str>("name")` by name. Strings are borrowed from the result, so reading a `&str` does not copy. Asking for the wrong type is a data error (GQLSTATUS `22G03`). Asking for a column the result does not have is `ZuError::InvalidArgument`, because that is a bug in the program and not something in the data. + +Transactions are statements. `START TRANSACTION`, then your writes, then `COMMIT` or `ROLLBACK`. `START TRANSACTION READ ONLY` is there too. A statement outside a transaction commits on its own. + +```rust +use zudb::{Database, params}; + +fn main() -> zudb::Result<()> { + let db = Database::memory()?; + let mut conn = db.connect()?; + + conn.execute("START TRANSACTION")?; + conn.execute("INSERT (a:person {uid: 1, name: 'ada'})-[:knows {since: 1843}]->(b:person {uid: 2, name: 'charles'})")?; + conn.execute("COMMIT")?; + + conn.execute("START TRANSACTION")?; + conn.execute("INSERT (p:person {uid: 3, name: 'nobody'})")?; + conn.execute("ROLLBACK")?; + + let (stmt, names) = conn.prepare( + "MATCH (a:person)-[k:knows]->(b:person) WHERE a.uid = $uid RETURN b.name AS name, k.since AS since", + )?; + assert_eq!(names, ["uid"]); + let rows = conn.execute_prepared(stmt, ¶ms! { "uid" => 1 })?; + for row in rows.iter() { + let name: &str = row.get_by_name("name")?; + let since: i64 = row.get_by_name("since")?; + println!("ada knows {name} since {since}"); + } + conn.close_prepared(stmt); + + let people = conn.query("MATCH (p:person) RETURN count(p) AS n")?; + let n: i64 = people.row(0)?.get_at(0)?; + println!("{n} people"); + Ok(()) +} +``` + +## Loading data + +Writing a row at a time through `INSERT` pays for a commit per statement. For bulk loads, use the appender. It buffers rows column by column in memory, and a flush writes them as sealed compressed segments with one small record in the log, however many rows there are. A row is every column of the table in declaration order, given as a tuple. + +```rust +use zudb::{Database, Flow}; + +fn main() -> zudb::Result<()> { + let db = Database::memory()?; + let mut conn = db.connect()?; + conn.execute("INSERT (p:person {uid: 0, name: 'first'})")?; + + let mut app = conn.appender("person")?; + for uid in 1..=10_000i64 { + app.append_row((uid, "someone"))?; + } + let loaded = app.close()?; + println!("appended {loaded}"); + + let mut total = 0i64; + conn.query_stream("MATCH (p:person) RETURN p.uid AS uid", &[], |batch| { + for row in batch.iter() { + total += row.get_at::(0)?; + } + Ok(Flow::More) + })?; + println!("sum {total}"); + + let plan = conn.explain("MATCH (p:person) WHERE p.uid > 9000 RETURN p.name AS name")?; + println!("{plan}"); + Ok(()) +} +``` + +You can also query columns you already hold in memory without loading them. `conn.register(frame)` makes a `Frame` visible as a table on that connection only, and nothing is copied. + +For files on disk, the `zu` command-line tool (crate `zudb-cli`) has `zu copy` for edge lists and labelled CSV datasets. With the `arrow` feature of this crate it also reads Parquet edge lists. + +## Errors + +Everything returns `zudb::Result`, which is `Result`. Query errors are `ZuError::Gql` and carry a diagnostic record with the standard GQLSTATUS code, a severity, a message, and the position in the statement text where that applies. `err.gqlstatus()` and `err.diagnostic()` read them. The other variants are for things that are not about the query: `Io`, `Corrupt` (a checksum or structure check failed), `Unsupported`, `InvalidArgument`, `Conflict` (lost a race to another writer), and `Interrupted`. Warnings that did not stop a statement come back in `QueryResult::notices` next to the rows. + +## Storage engines + +Storage is a trait (`GraphStore`, from `zudb-storage`) with three engines planned behind it. What you get through `Database` today is the native one. + +- `zu1` is a single `.zu1` file: columnar node groups, lightweight compressed segments, CSR adjacency for edges, a write-ahead log beside the file, and a dual header so a checkpoint is atomic. This is what `Database` opens and creates. +- `sqlite` maps the same graph onto an ordinary SQLite file. It is reachable through `zudb::sqlite::run` and the converters `zudb::convert::zu1_to_sqlite` and `sqlite_to_zu1`, and it is used as the reference engine the `zu1` results are tested against. It is not wired into `Database` yet. +- `s3` keeps the graph in object storage. Only the manifest format and the commit protocol exist so far, so there is nothing to open yet. + +## How it is put together + +This crate is the public API and the glue. The real work is split across the `zudb-*` crates, all versioned together: + +| Crate | Role | +| --- | --- | +| `zudb-common` | ids, `ZuError`, GQLSTATUS codes, logical types | +| `zudb-encoding` | FastLanes, ALP, FSST and the cascades over them | +| `zudb-storage` | the `GraphStore` trait and shared segment types | +| `zudb-zu1` | the native file format, log, MVCC epochs, bulk ingest | +| `zudb-sqlite`, `zudb-s3` | the other two engines | +| `zudb-vector` | typed column vectors, selection vectors, expression kernels | +| `zudb-query` | lexer, parser, binder, optimizer, plans, the row executor | +| `zudb-exec` | the push-based, morsel-parallel pipeline executor | + +A statement goes through them like this. The `Connection` owns a `Session`, which keeps the catalog, statistics and compiled plans resident. On a cache miss the text is parsed and bound against the catalog (`zudb-query`), then the optimizer picks join order and filter placement and produces a logical plan. The plan is cached by its text, so a warm statement is a hash lookup and a parameter bind. Execution goes to the pipeline executor in `zudb-exec` first. It splits the scan into morsels, pushes them through the operators on worker threads, and reads the file through a vectorized `Snapshot` that skips chunks using zone maps before decoding them. A plan shape it does not cover yet falls back to the older row executor in `zudb-query`, which must give exactly the same answer and is kept as a test oracle. Results are filled column by column and only turned into rows if you read rows. A write statement goes through the shared write side of the file (`write` and `shared` in this crate), which logs it, applies it to an in-memory overlay, and folds it into the file at checkpoint. + +The module docs on [docs.rs](https://docs.rs/zudb) go into each piece, and the design documents in the repository go further. + +## Features + +- `arrow`: Parquet edge list loading in the `zu1` ingest path. Off by default. Arrow export of query results is a separate crate, `zudb-arrow`. + +## License + +Apache-2.0 diff --git a/docs/api/model.json b/docs/api/model.json index 572ee7b9..0ca23321 100644 --- a/docs/api/model.json +++ b/docs/api/model.json @@ -1,7 +1,7 @@ { "schema": 1, "crate": "zu", - "version": "0.0.1", + "version": "0.0.2", "rustdoc_format_version": 61, "entities": [ { diff --git a/packaging/homebrew/zu.rb b/packaging/homebrew/zu.rb index 13993ac5..c8c8abdb 100644 --- a/packaging/homebrew/zu.rb +++ b/packaging/homebrew/zu.rb @@ -8,27 +8,27 @@ class Zu < Formula desc "Embedded property graph database with a GQL engine" homepage "https://github.com/tamnd/zu" - version "0.0.1" + version "0.0.2" license "Apache-2.0" on_macos do on_arm do - url "https://github.com/tamnd/zu/releases/download/v0.0.1/libzu-aarch64-apple-darwin.tar.zst" + url "https://github.com/tamnd/zu/releases/download/v0.0.2/libzu-aarch64-apple-darwin.tar.zst" sha256 "0000000000000000000000000000000000000000000000000000000000000000" end on_intel do - url "https://github.com/tamnd/zu/releases/download/v0.0.1/libzu-x86_64-apple-darwin.tar.zst" + url "https://github.com/tamnd/zu/releases/download/v0.0.2/libzu-x86_64-apple-darwin.tar.zst" sha256 "0000000000000000000000000000000000000000000000000000000000000000" end end on_linux do on_arm do - url "https://github.com/tamnd/zu/releases/download/v0.0.1/libzu-aarch64-unknown-linux-gnu.tar.zst" + url "https://github.com/tamnd/zu/releases/download/v0.0.2/libzu-aarch64-unknown-linux-gnu.tar.zst" sha256 "0000000000000000000000000000000000000000000000000000000000000000" end on_intel do - url "https://github.com/tamnd/zu/releases/download/v0.0.1/libzu-x86_64-unknown-linux-gnu.tar.zst" + url "https://github.com/tamnd/zu/releases/download/v0.0.2/libzu-x86_64-unknown-linux-gnu.tar.zst" sha256 "0000000000000000000000000000000000000000000000000000000000000000" end end diff --git a/packaging/scoop/zu.json b/packaging/scoop/zu.json index 94e59c1b..a2c71d86 100644 --- a/packaging/scoop/zu.json +++ b/packaging/scoop/zu.json @@ -1,13 +1,13 @@ { "##": "Generated by `cargo xtask packaging` from platforms.toml. Edit that, not this.", - "version": "0.0.1", + "version": "0.0.2", "description": "Embedded property graph database with a GQL engine", "homepage": "https://github.com/tamnd/zu", "license": "Apache-2.0", "depends": "zstd", "architecture": { "64bit": { - "url": "https://github.com/tamnd/zu/releases/download/v0.0.1/libzu-x86_64-pc-windows-msvc.tar.zst", + "url": "https://github.com/tamnd/zu/releases/download/v0.0.2/libzu-x86_64-pc-windows-msvc.tar.zst", "hash": "0000000000000000000000000000000000000000000000000000000000000000", "extract_dir": "libzu-x86_64-pc-windows-msvc" }