From 4885bb0a20aed8683d45dd32eb3299dfe3f491dc Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:55:55 +0900 Subject: [PATCH 1/8] test: define versioned CLI JSON v1 contract --- crates/zstdscope-cli/tests/json_cli.rs | 91 +++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/crates/zstdscope-cli/tests/json_cli.rs b/crates/zstdscope-cli/tests/json_cli.rs index cf92d95..232e1e1 100644 --- a/crates/zstdscope-cli/tests/json_cli.rs +++ b/crates/zstdscope-cli/tests/json_cli.rs @@ -5,7 +5,7 @@ use std::{ sync::atomic::{AtomicU64, Ordering}, }; -use serde_json::Value; +use serde_json::{Value, json}; const STANDARD_MAGIC: u32 = 0xFD2F_B528; const SKIPPABLE_MAGIC: u32 = 0x184D_2A55; @@ -15,20 +15,94 @@ const REFERENCE_COMPRESSED_CHECKSUM: &str = static NEXT_TEMP_FILE: AtomicU64 = AtomicU64::new(0); #[test] -fn json_standard_frame_has_explicit_snake_case_shape() { +fn json_v1_minimal_standard_frame_matches_schema_fixture() { + let output = run_json(&minimal_standard_frame()); + + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + let actual = parse_json(output.stdout); + + let expected = json!({ + "schema_version": 1, + "input_size": "9", + "frames": [ + { + "index": 0, + "span": { + "offset": "0", + "length": "9" + }, + "kind": { + "type": "standard", + "data": { + "magic_span": { + "offset": "0", + "length": "4" + }, + "header": { + "span": { + "offset": "4", + "length": "2" + }, + "descriptor": 0, + "descriptor_span": { + "offset": "4", + "length": "1" + }, + "window_descriptor_span": { + "offset": "5", + "length": "1" + }, + "frame_content_size": null, + "dictionary_id": null, + "window_size": "1024", + "content_checksum_flag": false, + "single_segment": false, + "unused_bit": false + }, + "blocks": [ + { + "index": 0, + "header_span": { + "offset": "6", + "length": "3" + }, + "content_span": { + "offset": "9", + "length": "0" + }, + "block_type": "raw", + "declared_size": 0, + "encoded_content_size": 0, + "is_last": true + } + ], + "content_checksum": null + } + } + } + ] + }); + + assert_eq!(actual, expected); +} + +#[test] +fn json_standard_frame_has_explicit_v1_snake_case_shape() { let output = run_json(&decode_hex(REFERENCE_COMPRESSED_CHECKSUM)); assert!(output.status.success()); assert!(output.stderr.is_empty()); let json = parse_json(output.stdout); - assert_eq!(json["input_size"], 63); + assert_eq!(json["schema_version"], 1); + assert_eq!(json["input_size"], "63"); assert!(json.get("inputSize").is_none()); assert_eq!(json["frames"][0]["index"], 0); assert_eq!(json["frames"][0]["kind"]["type"], "standard"); let standard = &json["frames"][0]["kind"]["data"]; - assert_eq!(standard["header"]["frame_content_size"]["value"], 10_240); + assert_eq!(standard["header"]["frame_content_size"]["value"], "10240"); assert!(standard["header"].get("frameContentSize").is_none()); assert_eq!(standard["blocks"][0]["block_type"], "compressed"); assert_eq!(standard["blocks"][0]["declared_size"], 49); @@ -45,8 +119,8 @@ fn json_preserves_dictionary_id_absent_vs_explicit_zero() { let explicit_zero = parse_json(run_json(&explicit_zero_dictionary_id_frame()).stdout); let dictionary_id = &explicit_zero["frames"][0]["kind"]["data"]["header"]["dictionary_id"]; assert_eq!(dictionary_id["encoded"], 0); - assert_eq!(dictionary_id["span"]["offset"], 6); - assert_eq!(dictionary_id["span"]["length"], 1); + assert_eq!(dictionary_id["span"]["offset"], "6"); + assert_eq!(dictionary_id["span"]["length"], "1"); } #[test] @@ -57,12 +131,13 @@ fn json_skippable_frame_uses_tagged_snake_case_shape() { assert!(output.stderr.is_empty()); let json = parse_json(output.stdout); + assert_eq!(json["schema_version"], 1); assert_eq!(json["frames"][0]["kind"]["type"], "skippable"); let skippable = &json["frames"][0]["kind"]["data"]; assert_eq!(skippable["variant"], 5); assert_eq!(skippable["declared_payload_size"], 2); - assert_eq!(skippable["payload_span"]["offset"], 8); - assert_eq!(skippable["payload_span"]["length"], 2); + assert_eq!(skippable["payload_span"]["offset"], "8"); + assert_eq!(skippable["payload_span"]["length"], "2"); } #[test] From 65a80e61b4c7e111605cdc1f550280f9641f9569 Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:04:26 +0900 Subject: [PATCH 2/8] feat: add dedicated CLI JSON v1 DTO --- Cargo.lock | 1 + crates/zstdscope-cli/Cargo.toml | 3 +- crates/zstdscope-cli/src/json.rs | 258 +++++++++++++++++++++++++++++++ crates/zstdscope-cli/src/main.rs | 3 +- 4 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 crates/zstdscope-cli/src/json.rs diff --git a/Cargo.lock b/Cargo.lock index bad5ef4..36825e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -251,6 +251,7 @@ name = "zstdscope-cli" version = "0.2.0" dependencies = [ "clap", + "serde", "serde_json", "zstdscope", ] diff --git a/crates/zstdscope-cli/Cargo.toml b/crates/zstdscope-cli/Cargo.toml index f0e5d55..7a04c3c 100644 --- a/crates/zstdscope-cli/Cargo.toml +++ b/crates/zstdscope-cli/Cargo.toml @@ -15,5 +15,6 @@ doc = false [dependencies] clap = { version = "4.6", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -zstdscope = { path = "../zstdscope", features = ["serde"] } +zstdscope = { path = "../zstdscope" } diff --git a/crates/zstdscope-cli/src/json.rs b/crates/zstdscope-cli/src/json.rs new file mode 100644 index 0000000..da7ef10 --- /dev/null +++ b/crates/zstdscope-cli/src/json.rs @@ -0,0 +1,258 @@ +use std::io::Write; + +use serde::Serialize; +use zstdscope::{ + Block, BlockType, ByteSpan, ContentChecksum, DictionaryId, Frame, FrameContentSize, + FrameHeader, FrameKind, SkippableFrame, StandardFrame, ZstdFile, +}; + +const SCHEMA_VERSION: u32 = 1; + +pub(super) fn write(writer: &mut W, file: &ZstdFile) -> Result<(), serde_json::Error> { + serde_json::to_writer_pretty(writer, &JsonZstdFileV1::from(file)) +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonZstdFileV1 { + schema_version: u32, + input_size: String, + frames: Vec, +} + +impl From<&ZstdFile> for JsonZstdFileV1 { + fn from(source: &ZstdFile) -> Self { + Self { + schema_version: SCHEMA_VERSION, + input_size: source.input_size.to_string(), + frames: source.frames.iter().map(JsonFrameV1::from).collect(), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonFrameV1 { + index: usize, + span: JsonByteSpanV1, + kind: JsonFrameKindV1, +} + +impl From<&Frame> for JsonFrameV1 { + fn from(source: &Frame) -> Self { + Self { + index: source.index, + span: source.span.into(), + kind: (&source.kind).into(), + } + } +} + +#[derive(Serialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case")] +enum JsonFrameKindV1 { + Standard(JsonStandardFrameV1), + Skippable(JsonSkippableFrameV1), +} + +impl From<&FrameKind> for JsonFrameKindV1 { + fn from(source: &FrameKind) -> Self { + match source { + FrameKind::Standard(frame) => Self::Standard(frame.into()), + FrameKind::Skippable(frame) => Self::Skippable(frame.into()), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonStandardFrameV1 { + magic_span: JsonByteSpanV1, + header: JsonFrameHeaderV1, + blocks: Vec, + content_checksum: Option, +} + +impl From<&StandardFrame> for JsonStandardFrameV1 { + fn from(source: &StandardFrame) -> Self { + Self { + magic_span: source.magic_span.into(), + header: (&source.header).into(), + blocks: source.blocks.iter().map(JsonBlockV1::from).collect(), + content_checksum: source + .content_checksum + .as_ref() + .map(JsonContentChecksumV1::from), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonFrameHeaderV1 { + span: JsonByteSpanV1, + descriptor: u8, + descriptor_span: JsonByteSpanV1, + window_descriptor_span: Option, + frame_content_size: Option, + dictionary_id: Option, + window_size: String, + content_checksum_flag: bool, + single_segment: bool, + unused_bit: bool, +} + +impl From<&FrameHeader> for JsonFrameHeaderV1 { + fn from(source: &FrameHeader) -> Self { + Self { + span: source.span.into(), + descriptor: source.descriptor, + descriptor_span: source.descriptor_span.into(), + window_descriptor_span: source.window_descriptor_span.map(JsonByteSpanV1::from), + frame_content_size: source + .frame_content_size + .as_ref() + .map(JsonFrameContentSizeV1::from), + dictionary_id: source + .dictionary_id + .as_ref() + .map(JsonDictionaryIdV1::from), + window_size: source.window_size.to_string(), + content_checksum_flag: source.content_checksum_flag, + single_segment: source.single_segment, + unused_bit: source.unused_bit, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonFrameContentSizeV1 { + value: String, + span: JsonByteSpanV1, +} + +impl From<&FrameContentSize> for JsonFrameContentSizeV1 { + fn from(source: &FrameContentSize) -> Self { + Self { + value: source.value.to_string(), + span: source.span.into(), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonDictionaryIdV1 { + encoded: u32, + span: JsonByteSpanV1, +} + +impl From<&DictionaryId> for JsonDictionaryIdV1 { + fn from(source: &DictionaryId) -> Self { + Self { + encoded: source.encoded, + span: source.span.into(), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonBlockV1 { + index: usize, + header_span: JsonByteSpanV1, + content_span: JsonByteSpanV1, + block_type: JsonBlockTypeV1, + declared_size: u32, + encoded_content_size: u32, + is_last: bool, +} + +impl From<&Block> for JsonBlockV1 { + fn from(source: &Block) -> Self { + Self { + index: source.index, + header_span: source.header_span.into(), + content_span: source.content_span.into(), + block_type: source.block_type.into(), + declared_size: source.declared_size, + encoded_content_size: source.encoded_content_size, + is_last: source.is_last, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum JsonBlockTypeV1 { + Raw, + Rle, + Compressed, +} + +impl From for JsonBlockTypeV1 { + fn from(source: BlockType) -> Self { + match source { + BlockType::Raw => Self::Raw, + BlockType::Rle => Self::Rle, + BlockType::Compressed => Self::Compressed, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonContentChecksumV1 { + span: JsonByteSpanV1, + value: u32, +} + +impl From<&ContentChecksum> for JsonContentChecksumV1 { + fn from(source: &ContentChecksum) -> Self { + Self { + span: source.span.into(), + value: source.value, + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonSkippableFrameV1 { + magic_span: JsonByteSpanV1, + magic: u32, + variant: u8, + size_field_span: JsonByteSpanV1, + declared_payload_size: u32, + payload_span: JsonByteSpanV1, +} + +impl From<&SkippableFrame> for JsonSkippableFrameV1 { + fn from(source: &SkippableFrame) -> Self { + Self { + magic_span: source.magic_span.into(), + magic: source.magic, + variant: source.variant, + size_field_span: source.size_field_span.into(), + declared_payload_size: source.declared_payload_size, + payload_span: source.payload_span.into(), + } + } +} + +#[derive(Serialize)] +#[serde(rename_all = "snake_case")] +struct JsonByteSpanV1 { + offset: String, + length: String, +} + +impl From for JsonByteSpanV1 { + fn from(source: ByteSpan) -> Self { + Self { + offset: source.offset.to_string(), + length: source.length.to_string(), + } + } +} diff --git a/crates/zstdscope-cli/src/main.rs b/crates/zstdscope-cli/src/main.rs index f633a59..f81eb71 100644 --- a/crates/zstdscope-cli/src/main.rs +++ b/crates/zstdscope-cli/src/main.rs @@ -1,5 +1,6 @@ #![forbid(unsafe_code)] +mod json; mod render; use std::{ @@ -121,7 +122,7 @@ fn read_input_with_limit(path: &Path, max_input_bytes: u64) -> Result, C fn write_inspection(writer: &mut W, file: &ZstdFile, json: bool) -> Result<(), CliError> { if json { - serde_json::to_writer_pretty(&mut *writer, file).map_err(CliError::Json)?; + json::write(&mut *writer, file).map_err(CliError::Json)?; writer.write_all(b"\n").map_err(CliError::Output)?; } else { render::render(writer, file).map_err(CliError::Output)?; From 4711723052573abb845c84c0ab4b6b225b326853 Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:06:43 +0900 Subject: [PATCH 3/8] style: apply rustfmt to JSON DTO --- crates/zstdscope-cli/src/json.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/zstdscope-cli/src/json.rs b/crates/zstdscope-cli/src/json.rs index da7ef10..bc7d216 100644 --- a/crates/zstdscope-cli/src/json.rs +++ b/crates/zstdscope-cli/src/json.rs @@ -113,10 +113,7 @@ impl From<&FrameHeader> for JsonFrameHeaderV1 { .frame_content_size .as_ref() .map(JsonFrameContentSizeV1::from), - dictionary_id: source - .dictionary_id - .as_ref() - .map(JsonDictionaryIdV1::from), + dictionary_id: source.dictionary_id.as_ref().map(JsonDictionaryIdV1::from), window_size: source.window_size.to_string(), content_checksum_flag: source.content_checksum_flag, single_segment: source.single_segment, From 927cf2ebdda188fabec1538bc1e1a19305313d2a Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:12:21 +0900 Subject: [PATCH 4/8] perf: box large JSON frame DTO variant --- crates/zstdscope-cli/src/json.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/zstdscope-cli/src/json.rs b/crates/zstdscope-cli/src/json.rs index bc7d216..3b85483 100644 --- a/crates/zstdscope-cli/src/json.rs +++ b/crates/zstdscope-cli/src/json.rs @@ -51,14 +51,14 @@ impl From<&Frame> for JsonFrameV1 { #[derive(Serialize)] #[serde(tag = "type", content = "data", rename_all = "snake_case")] enum JsonFrameKindV1 { - Standard(JsonStandardFrameV1), + Standard(Box), Skippable(JsonSkippableFrameV1), } impl From<&FrameKind> for JsonFrameKindV1 { fn from(source: &FrameKind) -> Self { match source { - FrameKind::Standard(frame) => Self::Standard(frame.into()), + FrameKind::Standard(frame) => Self::Standard(Box::new(frame.into())), FrameKind::Skippable(frame) => Self::Skippable(frame.into()), } } From bab4bb56a3d6e80fae2ed878ad6eb3ac5edad566 Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:15:13 +0900 Subject: [PATCH 5/8] test: define CLI distribution and supply-chain gates --- .github/workflows/ci.yml | 52 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e433e2b..190515e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,19 +10,25 @@ on: permissions: contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: quality: name: Quality gates runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install Rust stable uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 with: toolchain: stable components: rustfmt, clippy + targets: wasm32-unknown-unknown - name: Format run: cargo fmt --check @@ -41,18 +47,53 @@ jobs: - name: Core default features run: cargo test -p zstdscope --no-default-features --locked - - name: Package contents + - name: Core WebAssembly compile + run: cargo check -p zstdscope --target wasm32-unknown-unknown --no-default-features --locked + + - name: Core package contents run: cargo package -p zstdscope --list --locked - - name: Publish dry run + - name: Core publish dry run run: cargo publish -p zstdscope --dry-run --locked + - name: CLI package contents + run: cargo package -p zstdscope-cli --list --locked + + - name: CLI publish dry run + run: cargo publish -p zstdscope-cli --dry-run --locked + + - name: Packaged CLI smoke test + shell: bash + run: | + cargo package -p zstdscope-cli --locked + package_dir="target/package/zstdscope-cli-0.2.0" + install_root="$RUNNER_TEMP/zstdscope-install" + cargo install --path "$package_dir" --locked --root "$install_root" + + test "$("$install_root/bin/zstdscope" --version)" = "zstdscope 0.2.0" + printf '\x28\xb5\x2f\xfd\x00\x00\x01\x00\x00' > "$RUNNER_TEMP/minimal.zst" + "$install_root/bin/zstdscope" inspect "$RUNNER_TEMP/minimal.zst" + "$install_root/bin/zstdscope" inspect "$RUNNER_TEMP/minimal.zst" --json \ + | grep -F '"schema_version": 1' + + supply-chain: + name: Supply-chain policy + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Check advisories, licenses, bans, and sources + uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2 + msrv: name: MSRV (Rust 1.85.0) runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install Rust 1.85.0 uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 @@ -68,6 +109,7 @@ jobs: platform-tests: name: Tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -77,7 +119,7 @@ jobs: - macos-latest steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install Rust stable uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 From 9cf2b1504258b8797ee76717911990820002b23c Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:18:10 +0900 Subject: [PATCH 6/8] feat: publish CLI and enforce supply-chain policy --- .github/dependabot.yml | 13 +++++++++++++ crates/zstdscope-cli/Cargo.toml | 9 ++++++--- deny.toml | 25 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 deny.toml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..0c08621 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/crates/zstdscope-cli/Cargo.toml b/crates/zstdscope-cli/Cargo.toml index 7a04c3c..a2b4424 100644 --- a/crates/zstdscope-cli/Cargo.toml +++ b/crates/zstdscope-cli/Cargo.toml @@ -5,8 +5,11 @@ edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true -publish = false -description = "Command-line interface for ZstdScope." +readme.workspace = true +description = "Command-line structural inspector for Zstandard files." +documentation = "https://docs.rs/zstdscope" +keywords = ["zstd", "zstandard", "inspection", "compression", "cli"] +categories = ["command-line-utilities", "compression"] [[bin]] name = "zstdscope" @@ -17,4 +20,4 @@ doc = false clap = { version = "4.6", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -zstdscope = { path = "../zstdscope" } +zstdscope = { version = "0.2.0", path = "../zstdscope" } diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..5e2470b --- /dev/null +++ b/deny.toml @@ -0,0 +1,25 @@ +[graph] +all-features = true + +[advisories] +ignore = [] + +[licenses] +allow = [ + "Apache-2.0", + "MIT", + "Unicode-3.0", + "Unlicense", +] +confidence-threshold = 0.8 + +[bans] +multiple-versions = "warn" +wildcards = "deny" +highlight = "all" + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +allow-git = [] From 38f017050b53731e4334d4ff9d2b3cf0c3407000 Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:28:17 +0900 Subject: [PATCH 7/8] docs: record JSON, distribution, and CI policies --- .github/workflows/ci.yml | 7 +- README.ja.md | 212 +++++++++++++++++++--------- README.md | 75 ++++++---- ROADMAP.md | 23 ++- docs/SUPPLY-CHAIN.md | 131 +++++++++++++++++ docs/adr/0005-versioned-cli-json.md | 92 ++++++++++++ docs/adr/0006-cli-distribution.md | 95 +++++++++++++ 7 files changed, 533 insertions(+), 102 deletions(-) create mode 100644 docs/SUPPLY-CHAIN.md create mode 100644 docs/adr/0005-versioned-cli-json.md create mode 100644 docs/adr/0006-cli-distribution.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 190515e..c2beafa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,11 +66,14 @@ jobs: shell: bash run: | cargo package -p zstdscope-cli --locked - package_dir="target/package/zstdscope-cli-0.2.0" + package_dir="$(find target/package -maxdepth 1 -type d -name 'zstdscope-cli-*' -print -quit)" + test -n "$package_dir" + + package_version="${package_dir##*-}" install_root="$RUNNER_TEMP/zstdscope-install" cargo install --path "$package_dir" --locked --root "$install_root" - test "$("$install_root/bin/zstdscope" --version)" = "zstdscope 0.2.0" + test "$("$install_root/bin/zstdscope" --version)" = "zstdscope $package_version" printf '\x28\xb5\x2f\xfd\x00\x00\x01\x00\x00' > "$RUNNER_TEMP/minimal.zst" "$install_root/bin/zstdscope" inspect "$RUNNER_TEMP/minimal.zst" "$install_root/bin/zstdscope" inspect "$RUNNER_TEMP/minimal.zst" --json \ diff --git a/README.ja.md b/README.ja.md index a9afb27..79a13b8 100644 --- a/README.ja.md +++ b/README.ja.md @@ -5,16 +5,18 @@ [![CI](https://github.com/tappe9/zstdscope/actions/workflows/ci.yml/badge.svg)](https://github.com/tappe9/zstdscope/actions/workflows/ci.yml) [![License](https://img.shields.io/crates/l/zstdscope.svg)](https://github.com/tappe9/zstdscope#license) -ZstdScope は、Zstandard圧縮データ形式の構造を解析・可視化するための **Pure Rust製Inspector / Parserツールキット**を目指すOSSプロジェクトです。 +ZstdScopeは、Zstandard圧縮データ形式の構造を解析する **Pure Rust製Inspector / Parserツールキット**です。 -このプロジェクトの主目的は圧縮・展開ではなく、`.zst`などのZstandardストリームに含まれるFrame、Header、Block、offset、sizeなどの構造情報を安全に取得できるAPIを提供することです。 +主目的は圧縮・展開ではなく、`.zst`などのZstandard streamに含まれるFrame、Header、Block、offset、sizeなどのencoded structureを、安全かつ再利用可能なRust APIとCLIから取得することです。Compressed Blockのpayload内部はopaqueとして扱います。 -> **v0.2.0を[crates.io](https://crates.io/crates/zstdscope)で公開しています。** pre-1.0のためPublic APIとJSON representationは今後意図的に変更される可能性があります。 +> **`zstdscope` library v0.2.0を[crates.io](https://crates.io/crates/zstdscope)で公開しています。** pre-1.0のため、Rust Public APIとversioned CLI JSON contractは、明示的な方針に基づいて変更される可能性があります。 **リンク:** [crates.io](https://crates.io/crates/zstdscope) · [docs.rs](https://docs.rs/zstdscope) · [v0.2.0 Release](https://github.com/tappe9/zstdscope/releases/tag/v0.2.0) ## インストール +### Rust library + Rustプロジェクトへ追加する場合: ```bash @@ -28,60 +30,97 @@ cargo add zstdscope zstdscope = "0.2" ``` -Serdeによるserializationを有効にする場合: +Public inspection modelのSerde serializationを有効にする場合: ```bash cargo add zstdscope --features serde ``` -## v0.1で解析する範囲 +### CLI -- Standard Zstandard magic number -- 全16種類のSkippable Frame magic number -- Frame Header Descriptor -- Window Size -- Frame Content Size -- Dictionary ID - - フィールドなし - - 明示的に`0`が格納されている - - 非0のID - を区別する -- Content Checksumの有無と格納値 -- Block Header -- Raw / RLE / Compressed Block -- Last Block Flag -- Frame / Header field / Blockのsource spanとencoded size -- 複数Frameの連結ストリーム +CLI package名は`zstdscope-cli`、installされるbinary名は`zstdscope`です。公開済みCLI releaseはcrates.ioからinstallします。 -v0.1ではCompressed Block内部の以下は解析しません。 +```bash +cargo install zstdscope-cli --locked +``` -- Literals -- Sequences -- Huffman -- FSE +CLI crateの初回release公開前に現在の`main`を利用する場合は、repositoryを手動cloneせず次のコマンドでinstallできます。 -また、Content Checksumは格納値と位置を取得しますが、復元データに対する検証は行いません。圧縮・展開機能もZstdScopeの目的外です。 +```bash +cargo install --git https://github.com/tappe9/zstdscope zstdscope-cli --locked +``` -v0.1は**実装している構造境界についてstrict**です。空入力・途中で切れたデータ・未知のtop-level magic・reserved bit/type・不可能なblock size・検証可能なFrame Content Size矛盾・末尾の不完全なFrameはエラーとします。一方、opaqueとして扱うCompressed Block内部やContent Checksumの正当性まで検証済みであるとは扱いません。 +正式なCLI release channelにはcrates.ioを採用します。GitHub Release向けprebuilt binary、署名、checksumは現在提供しません。これらは再現可能な自動release policyを別途定義した場合のみ追加します。 -## 構成 +Source buildはGitHub-hosted runner上のUbuntu x86_64、Windows x86_64、macOS arm64で継続検証します。その他のRust対応targetはbest effortです。これはsource buildのsupport範囲であり、prebuilt binary提供を保証するものではありません。 -```text -zstdscope/ -├── crates/ -│ ├── zstdscope/ # Pure Rust Parser Library -│ └── zstdscope-cli/ # CLI -├── docs/ -└── ARCHITECTURE.md -``` +## 解析する範囲 + +現在のstructural scopeでは以下を扱います。 -Public APIは次の形です。 +- Standard Frameと全16種類のSkippable Frame magic +- 複数Frameの連結streamと正確なFrame境界 +- Frame Header Descriptorとderived Window Size +- 全encoded widthのFrame Content Size +- Block-level decoded-size boundsから検証可能なFrame Content Size矛盾 +- Dictionary IDのfieldなし・明示的な`0`・非0の区別 +- Raw / RLE / Compressed Block Headerとencoded content span +- RLEのdeclared sizeと1 byteのencoded content sizeの区別 +- 必須outer section headerを格納できないCompressed Blockのstructural rejection +- Content Checksumの格納値とspan(checksum validationは行わない) +- major fieldのzero-based source span +- malformed / truncated inputに対するlocation-aware typed error + +Literals、Sequences、Huffman、FSEなどCompressed Block内部は解析しません。Parserは最低1つのcomplete Frameを要求し、入力全体をconsumeします。空入力、未知のtop-level magic、reserved encoding、不可能なstructural size、検証可能な矛盾、末尾のpartial Frameはerrorです。 + +## Rust library + +主なAPIは次のとおりです。 ```rust pub fn inspect(data: &[u8]) -> Result; ``` -`inspect()`は従来どおりFrame/Block件数に上限を設けない簡潔なAPIです。外部から受け取った入力などにmetadata件数の上限を設けたい場合は、`inspect_with_limits()`を利用できます。 +使用例: + +```rust +use zstdscope::{FrameKind, inspect}; + +fn main() -> Result<(), Box> { + let bytes = std::fs::read("sample.zst")?; + let file = inspect(&bytes)?; + + for frame in &file.frames { + match &frame.kind { + FrameKind::Standard(standard) => { + println!( + "frame #{}: standard, {} blocks, offset={}, size={}", + frame.index, + standard.blocks.len(), + frame.span.offset, + frame.span.length + ); + } + FrameKind::Skippable(skippable) => { + println!( + "frame #{}: skippable variant {}, payload={} bytes", + frame.index, + skippable.variant, + skippable.declared_payload_size + ); + } + } + } + + Ok(()) +} +``` + +Public offsetはencoded input先頭からのzero-based byte offsetです。Opaque BlockとSkippable payloadは解析結果へcopyせず、spanで表現します。 + +### Resource limits + +`inspect()`はFrame/Block件数に上限を設けない簡潔なAPIです。外部入力などにmetadata件数のbudgetを設ける場合は`inspect_with_limits()`を利用します。 ```rust use zstdscope::{InspectionLimits, inspect_with_limits}; @@ -95,51 +134,88 @@ let limits = InspectionLimits { let file = inspect_with_limits(&bytes, limits)?; ``` -上記の値は例であり、すべての用途に対する安全なdefault値ではありません。用途と想定入力に応じて設定してください。設定値ちょうどまでは許可され、さらに1件のFrame/Blockを解析しようとした時点で、対象構造の開始offsetを持つ`ZstdError::ResourceLimitExceeded`を返します。 +上記は例であり、すべての用途に対する安全なdefault値ではありません。設定値ちょうどまでは許可され、さらに1件を解析しようとした時点で、対象構造の開始offsetを持つ`ZstdError::ResourceLimitExceeded`を返します。 -この制限はFrame/Block metadataの**件数**を制御するもので、callerが保持する入力sliceのbyte数を制限したりstreaming化したりするものではありません。Block/Skippable payloadを解析結果へコピーしない既存方針も維持します。 +この制限はFrame/Block metadataの件数を制御するもので、callerが保持する入力sliceのbyte数を制限したりstreaming化したりするものではありません。 -Public API方針は[Public API設計](docs/API-DESIGN.md)で管理しています。 +### Optional serialization + +Core crateのSerdeはoptionalです。 + +```toml +[features] +default = [] +serde = ["dep:serde"] +``` + +このfeatureで得られるPublic Rust modelのserialization表現と、CLI JSON wire contractは独立した契約です。Parseのみを行うconsumerにSerde dependencyは不要です。 ## CLI +Human-readable output: + ```text zstdscope inspect sample.zst -zstdscope inspect sample.zst --json ``` -リポジトリから実行する場合: +Repository checkoutから実行する場合: ```text cargo run -p zstdscope-cli -- inspect sample.zst -cargo run -p zstdscope-cli -- inspect sample.zst --json ``` +Frame typeと境界、Header metadata、Block type/size、Skippable payload metadata、格納されているChecksum metadataを表示します。 + ### 大きな入力ファイル -CLIは引き続きin-memoryのlibrary APIを利用するため、受け入れた入力ファイルは解析中にメモリ上へ保持されます。defaultのCLI動作をboundedにするため、`zstdscope inspect`は **268,435,456 bytes(256 MiB)** を超えるencoded inputをparse前に拒否します。 +CLIはin-memoryのlibrary APIを利用するため、受け入れた入力ファイルは解析中にメモリ上へ保持されます。defaultのCLI動作をboundedにするため、`zstdscope inspect`は **268,435,456 bytes(256 MiB)** を超えるencoded inputをparse前に拒否します。 -上限を明示的に変更する場合は`--max-input-bytes`を指定します。 +上限を明示的に変更する場合: ```text zstdscope inspect large.zst --max-input-bytes 1073741824 ``` -上限を引き上げると、encoded input bufferに対して許容する最大メモリ量も増えます。CLIは可能な場合は全体read前にfile sizeを確認し、実際のread自体にも上限を適用するため、read中にファイルが増加しても設定した上限をそのまま通過しません。 +上限を引き上げるとencoded input bufferへ許容する最大メモリ量も増えます。CLIは可能な場合は全体read前にfile sizeを確認し、実際のread自体にも上限を適用するため、read中にファイルが増加しても設定上限をsilently bypassしません。 + +CLIのbyte上限と、libraryの`inspect_with_limits()`によるmetadata件数上限は別のresource policyです。どちらもstreaming化するものではありません。 + +### Versioned JSON output + +Machine-readable output: + +```text +zstdscope inspect sample.zst --json +``` + +CLIはPublic Rust modelを直接serializeせず、専用JSON DTOを利用します。現在のwire contractは次のとおりです。 -このCLIのbyte上限と、libraryの`inspect_with_limits()`によるFrame/Block metadata件数上限は別のresource policyです。どちらもstreaming化するものではありません。入力全体をメモリへ保持したくない非常に大きなファイルについては、将来のstreaming/file-backed library APIで対応する方針です。 +- top-levelに`"schema_version": 1` +- field名とenum値は明示的な`snake_case` +- Frame kindは`type` / `data`のtagged representation +- Rustの`u64`値(offset、length、input size、window size、Frame Content Sizeなど)はJavaScriptでinteger precisionを失わないdecimal string +- boundedな`u8`、`u32`、`usize`はJSON number +- Dictionary IDのfieldなしと明示的な`0`を保持 +- RLEのdeclared sizeとencoded sizeを保持 -I/O error、parse error、input size limit超過は非0で終了し、diagnosticをstderrへ出力します。pipe先が通常どおり先に閉じた場合のbroken pipeはpanicさせず正常終了として扱います。 +1.0より前でも、既存consumerと互換なadditive fieldはschema version 1内で追加できます。fieldの削除・rename・type変更、enum/tag representation変更などのbreaking changeでは`schema_version`を更新し、release noteへ記録します。Human-readable outputは別契約であり、JSON DTO refactorの影響を受けません。 -再利用可能な`zstdscope` libraryは[crates.io](https://crates.io/crates/zstdscope)で公開済みです。API documentationは[docs.rs](https://docs.rs/zstdscope)で確認できます。 +I/O error、parse error、input size limit超過は非0で終了し、diagnosticをstderrへ出力します。Partial-success JSONは出力しません。Output write failureでpanicせず、downstream processが通常どおりpipeを閉じた場合は正常終了として扱います。 -`zstdscope-cli` package自体は現在`publish = false`で、CLI binaryの配布はlibrary crateとは分けて今後整備します。 +詳細は[ADR 0005](docs/adr/0005-versioned-cli-json.md)と[ADR 0006](docs/adr/0006-cli-distribution.md)を参照してください。 -## JSON +## 構成 -CLIでは`--json`によるmachine-readableな解析結果を提供します。 +```text +zstdscope/ +├── crates/ +│ ├── zstdscope/ # Pure Rust Parser Library +│ └── zstdscope-cli/ # Public library APIを利用するCLI +├── docs/ +└── ARCHITECTURE.md +``` -JSONのfield名とenum値は`snake_case`を使用し、serialization libraryのdefault挙動に偶然依存しないよう、表現を明示してテストしています。Dictionary IDの「fieldなし」と「明示的な0」、RLEのdeclared sizeとencoded sizeなど、Inspector固有の情報も保持します。 +Public API方針は[Public API設計](docs/API-DESIGN.md)で管理しています。 ## ドキュメント @@ -147,31 +223,29 @@ JSONのfield名とenum値は`snake_case`を使用し、serialization libraryのd - [アーキテクチャ](ARCHITECTURE.md) - [Zstandardフォーマットメモ](docs/ZSTD-FORMAT.md) - [Public API設計](docs/API-DESIGN.md) +- [Supply-chain policy](docs/SUPPLY-CHAIN.md) +- [Fuzzing guide](FUZZING.md) - [Roadmap](ROADMAP.md) - [ADR](docs/adr/) 設計ドキュメントは英語を正式版として管理し、このREADMEは日本語で概要を確認するためのものとします。 -## 仕様の参照先 - -- [RFC 8878](https://www.rfc-editor.org/rfc/rfc8878.html) -- [Zstandard reference format specification](https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md) -- [Zstandard reference implementation](https://github.com/facebook/zstd) - ## 開発・安全性方針 -ZstdScopeでは任意の入力byte列を信用しません。 - -Parserでは特に以下を重視します。 +ZstdScopeは任意の入力byte列を信用しません。 -- bounds check +- bounds-checked read/skip - checked arithmetic - malformed inputでparser panicしない -- 不正なsizeを理由にopaque payloadをコピーしない -- v0.1ではproject codeに`unsafe`を使用しない -- Zstdの公開仕様を根拠に実装する +- opaque payloadを結果へcopyしない +- core crateでproject-authored `unsafe`を禁止 +- Zstandard公開仕様を根拠に実装 +- configurable metadata count limits +- CLI default 256 MiB encoded-input guard + +Public parser APIは入力全体をメモリ上で扱います。`inspect_with_limits()`はmetadata件数を制限し、CLIは別途encoded-input byte上限を持ちます。`--max-input-bytes`で上限を変更できますが、受け入れた入力は依然として全体がメモリ上に存在します。Streaming/file-backed APIは後続milestoneです。 -Public parser APIは引き続き入力全体をメモリ上で扱います。`inspect_with_limits()`はFrame/Block metadata件数を制限し、CLIはそれとは別にdefault 256 MiBのencoded-input guardを持ちます。`--max-input-bytes`で明示的に上限を変更できますが、受け入れたCLI入力は依然として全体がメモリ上に存在します。streaming/file-backed APIは後続milestoneで整備します。 +CIではformat、Clippy、test、rustdoc、MSRV、Ubuntu/Windows/macOS、package/publish dry-run、packaged CLI smoke、WebAssembly compile、advisory/license/source policyを検証します。詳細は[Supply-chain policy](docs/SUPPLY-CHAIN.md)と[SECURITY.md](SECURITY.md)を参照してください。 ## ライセンス diff --git a/README.md b/README.md index 8f1f311..2b1060f 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,14 @@ ZstdScope is a pure-Rust parser and structural inspection toolkit for the Zstand The project is intentionally focused on **inspection**, not compression or decompression. It exposes encoded structure and source-byte metadata through a reusable Rust API and a CLI, while leaving compressed block payloads opaque. -> **v0.2.0 is available on [crates.io](https://crates.io/crates/zstdscope).** ZstdScope remains pre-1.0, so the Rust API and JSON representation may still evolve intentionally. +> **v0.2.0 of the `zstdscope` library is available on [crates.io](https://crates.io/crates/zstdscope).** ZstdScope remains pre-1.0, so the Rust API and versioned CLI JSON contract may still evolve intentionally. **Links:** [crates.io](https://crates.io/crates/zstdscope) · [docs.rs](https://docs.rs/zstdscope) · [v0.2.0 release](https://github.com/tappe9/zstdscope/releases/tag/v0.2.0) ## Installation +### Rust library + Add ZstdScope to a Rust project: ```bash @@ -34,9 +36,27 @@ For optional Serde serialization support: cargo add zstdscope --features serde ``` +### CLI + +The supported CLI package is named `zstdscope-cli`; it installs the `zstdscope` binary. Released CLI versions use crates.io: + +```bash +cargo install zstdscope-cli --locked +``` + +Until the first CLI crate release is published, install the current `main` revision without manually cloning the repository: + +```bash +cargo install --git https://github.com/tappe9/zstdscope zstdscope-cli --locked +``` + +The crates.io package is the selected release distribution channel. Prebuilt GitHub Release binaries, signatures, and checksums are not currently provided; they may be added only with an explicit, automated release policy. + +Source builds are continuously tested on GitHub-hosted Ubuntu x86_64, Windows x86_64, and macOS arm64 runners. Other Rust-supported targets are best effort. This is a source-build support statement, not a promise of prebuilt binaries for those targets. + ## What ZstdScope inspects -The current v0.1 scope includes: +The current structural scope includes: - Standard and all 16 Skippable Frame magic values; - concatenated frames with exact frame boundaries; @@ -50,9 +70,7 @@ The current v0.1 scope includes: - zero-based source spans for major encoded fields; - typed, location-aware parse errors for malformed and truncated input. -Parsing literals, sequences, Huffman tables, FSE tables, and other compressed-block internals is intentionally deferred beyond v0.1. - -The parser is strict for the structural envelope implemented by v0.1: it requires at least one complete frame and consumes the entire input. Empty input, malformed structures that v0.1 can validate, unknown top-level magic, reserved encodings, impossible structural sizes, and trailing partial frames are errors. Compressed-block internals and content-checksum validity are intentionally not validated in v0.1. +Parsing literals, sequences, Huffman tables, FSE tables, and other compressed-block internals is intentionally deferred. The parser requires at least one complete frame and consumes the entire input. Empty input, malformed structures that the current structural layer can validate, unknown top-level magic, reserved encodings, impossible structural sizes, and trailing partial frames are errors. Compressed-block internals and content-checksum validity are intentionally not validated. ## Rust library @@ -101,7 +119,7 @@ All public offsets are zero-based byte offsets into the encoded input. Opaque bl ### Configurable resource limits -`inspect()` intentionally preserves the simple, unlimited frame/block-count behavior. Applications that inspect untrusted or externally supplied inputs can instead apply explicit metadata budgets with `inspect_with_limits()`: +`inspect()` preserves the simple, unlimited frame/block-count behavior. Applications that inspect untrusted or externally supplied inputs can instead apply explicit metadata budgets with `inspect_with_limits()`: ```rust use zstdscope::{InspectionLimits, inspect_with_limits}; @@ -129,7 +147,7 @@ default = [] serde = ["dep:serde"] ``` -Enabling the `serde` feature adds `Serialize` support to the public inspection model. Parsing-only users do not require Serde. +Enabling the `serde` feature adds `Serialize` support to the public inspection model. This library representation is independent from the CLI JSON wire contract. Parsing-only users do not require Serde. ## CLI @@ -149,7 +167,7 @@ The output reports frame type and boundaries, header metadata, block types and s ### Large input files -The CLI still uses the in-memory library API, so an accepted input file is resident in memory while it is inspected. To keep the default CLI behavior bounded, `zstdscope inspect` rejects encoded input larger than **268,435,456 bytes (256 MiB)** before parsing. +The CLI uses the in-memory library API, so an accepted input file is resident in memory while it is inspected. To keep default behavior bounded, `zstdscope inspect` rejects encoded input larger than **268,435,456 bytes (256 MiB)** before parsing. Raise or lower that boundary explicitly with `--max-input-bytes`: @@ -157,11 +175,11 @@ Raise or lower that boundary explicitly with `--max-input-bytes`: zstdscope inspect large.zst --max-input-bytes 1073741824 ``` -Raising the limit also raises the maximum memory commitment for the encoded input buffer. The CLI checks the file size before the full read when possible and also bounds the actual read, so a file that grows while being read cannot silently bypass the configured limit. +Raising the limit also raises the maximum memory commitment for the encoded input buffer. The CLI checks file size before the full read when possible and also bounds the actual read, so a file that grows while being read cannot silently bypass the configured limit. This CLI byte limit is separate from `inspect_with_limits()`, which only limits frame/block metadata counts in the library. Neither mechanism makes inspection streaming. A future streaming/file-backed library API remains the path for files that should not be held fully in memory. -### JSON output +### Versioned JSON output Use `--json` for machine-readable output: @@ -169,21 +187,21 @@ Use `--json` for machine-readable output: zstdscope inspect sample.zst --json ``` -or from the workspace: - -```text -cargo run -p zstdscope-cli -- inspect sample.zst --json -``` - -JSON field names and serialized enum values use explicit `snake_case` representations. `FrameKind` uses a tagged `type` / `data` shape, and Inspector-specific distinctions such as absent versus explicitly encoded zero Dictionary IDs and RLE declared versus encoded sizes are preserved. +The CLI emits a dedicated JSON DTO rather than serializing the public Rust model directly. The current wire contract has: -I/O and parse failures return a non-zero exit status, write diagnostics to stderr, and do not emit partial-success JSON. An input that exceeds the CLI byte limit also returns a non-zero exit status with a structured CLI error. Output write failures are handled without panicking; a downstream process closing a pipe normally is treated as a normal CLI termination. +- top-level `"schema_version": 1`; +- explicit `snake_case` field names and enum values; +- a tagged `type` / `data` frame-kind representation; +- decimal strings for every Rust `u64` value, including offsets, lengths, input size, window size, and Frame Content Size, so JavaScript consumers do not lose integer precision; +- JSON numbers for bounded `u8`, `u32`, and `usize` fields; +- preserved distinctions between absent and explicitly encoded zero Dictionary IDs; +- preserved RLE declared-size versus encoded-size semantics. -### Release distribution +Before 1.0, backward-compatible additive fields may remain within schema version 1. Removing or renaming fields, changing field types, changing enum/tag representations, or otherwise breaking existing consumers requires a new `schema_version` and release-note documentation. Human-readable output is a separate contract and is unaffected by JSON DTO refactors. -The reusable `zstdscope` Rust library is published on [crates.io](https://crates.io/crates/zstdscope), with API documentation on [docs.rs](https://docs.rs/zstdscope). +I/O and parse failures return a non-zero exit status, write diagnostics to stderr, and do not emit partial-success JSON. An input that exceeds the CLI byte limit also returns a non-zero exit status with a structured CLI error. Output write failures are handled without panicking; a downstream process closing a pipe normally is treated as normal CLI termination. -The `zstdscope-cli` package currently has `publish = false`; CLI binary distribution is intentionally separate and may be added through GitHub Releases or another documented channel later. +See [ADR 0005](docs/adr/0005-versioned-cli-json.md) for the JSON decision and [ADR 0006](docs/adr/0006-cli-distribution.md) for distribution policy. ## Workspace @@ -196,7 +214,7 @@ zstdscope/ └── ARCHITECTURE.md ``` -The accepted v0.1 API direction is documented in [Public API design](docs/API-DESIGN.md). +The accepted API direction is documented in [Public API design](docs/API-DESIGN.md). ## Design principles @@ -206,9 +224,9 @@ ZstdScope aims to: - preserve byte-level distinctions useful to inspection and hex-viewer tooling; - provide precise diagnostics for malformed or unsupported input; - remain safe on untrusted byte input within the documented in-memory/resource model; -- keep parsing independent from filesystem, terminal, and CLI concerns; +- keep parsing independent from filesystem, terminal, JSON DTO, and CLI concerns; - keep mandatory core dependencies small; -- remain suitable for a future `wasm32` target. +- remain suitable for `wasm32-unknown-unknown` compilation. ## Non-goals @@ -217,8 +235,8 @@ ZstdScope is not intended to be: - a compressor; - a decompressor; - a replacement for the official `zstd` CLI or `libzstd`; -- a decoder for compressed block internals in v0.1; -- a content-checksum verifier in v0.1. +- a decoder for compressed block internals in the current structural scope; +- a content-checksum verifier. ## Documentation @@ -226,6 +244,7 @@ ZstdScope is not intended to be: - [Architecture](ARCHITECTURE.md) - [Zstandard format notes](docs/ZSTD-FORMAT.md) - [Public API design](docs/API-DESIGN.md) +- [Supply-chain policy](docs/SUPPLY-CHAIN.md) - [Fuzzing guide](FUZZING.md) - [Roadmap](ROADMAP.md) - [Architecture decision records](docs/adr/) @@ -244,11 +263,11 @@ Where the current reference specification and the RFC differ, the difference mus ZstdScope treats every input byte as untrusted. Parser reads and skips are bounds-checked, offset/size arithmetic is checked, opaque payloads are not copied into the inspection model, and the project forbids authored `unsafe` Rust in the core crate. -The public parser APIs remain intentionally in-memory. `inspect_with_limits()` can bound frame/block metadata counts for untrusted inputs, while `inspect()` retains the original unlimited count behavior. The CLI adds a separate default 256 MiB encoded-input guard with `--max-input-bytes` as an explicit override, but any accepted CLI input is still held fully in memory. Streaming/file-backed inspection remains later hardening work. +The public parser APIs remain intentionally in-memory. `inspect_with_limits()` can bound frame/block metadata counts for untrusted inputs, while `inspect()` retains unlimited count behavior. The CLI adds a separate default 256 MiB encoded-input guard with `--max-input-bytes` as an explicit override, but any accepted CLI input is still held fully in memory. Streaming/file-backed inspection remains later hardening work. Parser fuzzing is available through `cargo-fuzz`; successful fuzz parses are also checked against structural model invariants. See [FUZZING.md](FUZZING.md) for setup, execution, and regression-handling instructions. Fuzzing is manual initially and is not part of normal pull-request CI. -See [SECURITY.md](SECURITY.md) for the project security policy. +CI enforces formatting, Clippy, tests, rustdoc, MSRV, Ubuntu/Windows/macOS tests, package/publish dry-runs, packaged CLI smoke tests, a WebAssembly compile check, and the documented advisory/license/source policy. See [Supply-chain policy](docs/SUPPLY-CHAIN.md) and [SECURITY.md](SECURITY.md). ## Contributing diff --git a/ROADMAP.md b/ROADMAP.md index 40fb09a..b69e07e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -52,13 +52,29 @@ Released scope: - a manual `cargo-fuzz` target with structural model-invariant checks on successful parses; - bounded CLI file reads with a default 256 MiB encoded-input guard and explicit `--max-input-bytes` override. +## Post-v0.2 maintenance — Contract and distribution hardening + +Status: **completed on `main`; release publication is a separate release operation.** + +Completed scope: + +- dedicated CLI JSON schema-version-1 DTOs, decoupled from the public Rust model; +- decimal-string representation for JSON values originating from Rust `u64` fields; +- explicit JSON compatibility and breaking-change policy; +- publishable `zstdscope-cli` package with crates.io as the selected release channel; +- package, publish-dry-run, install, version, text-output, and JSON smoke validation from the generated CLI package; +- immutable GitHub Actions pins, least-privilege permissions, concurrency controls, and job timeouts; +- documented `cargo-deny` advisory/license/source policy; +- weekly Cargo and GitHub Actions Dependabot updates; +- `wasm32-unknown-unknown` compile-only coverage for the dependency-light core boundary. + Deferred candidate scope for later releases: - richer field-level spans for hex-view mapping; - more detailed diagnostics; - streaming or file-backed library inspection API for very large files; - scheduled or continuous fuzzing automation; -- JSON schema documentation; +- prebuilt, checksummed, and signed GitHub Release binaries; - CLI output refinements. ## v0.3 — Compressed-block structural metadata @@ -70,7 +86,7 @@ Candidate scope: - nested spans within Compressed blocks; - structural validation that does not require full decompression. -This milestone should not be started until the v0.1 frame/block model proves extensible enough to represent nested structures cleanly. +This milestone should not be started until the frame/block model proves extensible enough to represent nested structures cleanly. ## v0.4 — Entropy metadata inspection @@ -115,7 +131,8 @@ Potential criteria: - large corpus of valid and malformed tests; - mature diagnostics; - clear support policy for Zstandard format revisions; -- published crate and release artifacts. +- published crate and release artifacts; +- reconsidered mandatory semantic-version compatibility checks. ## Guiding rule diff --git a/docs/SUPPLY-CHAIN.md b/docs/SUPPLY-CHAIN.md new file mode 100644 index 0000000..515a416 --- /dev/null +++ b/docs/SUPPLY-CHAIN.md @@ -0,0 +1,131 @@ +# Supply-chain and CI policy + +Status: **Active** + +This document defines the dependency and workflow policy enforced for ZstdScope pull requests and `main`. + +## Goals + +The policy should detect actionable risk without turning CI into an undocumented or noisy gate. Every required check must have a clear owner, configuration, and exception process. + +## GitHub Actions + +- Workflow permissions default to `contents: read`. +- Third-party actions are pinned to immutable commit SHAs where practical. A human-readable release tag is retained as an inline comment. +- Superseded pull-request runs are cancelled through workflow concurrency controls; `main` runs are not cancelled. +- Every job has a bounded `timeout-minutes` value. +- Dependabot checks Cargo and GitHub Actions dependencies weekly. + +A dependency-update PR must still pass the complete quality and platform matrix. An automated update is not trusted merely because it was opened by Dependabot. + +## Cargo dependency policy + +`cargo-deny` is the required consolidated gate for: + +- RustSec advisories; +- allowed licenses; +- duplicate/wildcard dependency policy; +- registry and Git source policy. + +The configuration lives in `deny.toml` and is reviewed like production code. + +### Advisories + +No advisory is ignored by default. A temporary ignore requires: + +1. a linked issue describing reachability and impact; +2. a reason an upgrade or removal is not currently possible; +3. a bounded removal condition or date; +4. explicit review in the pull request that changes `deny.toml`. + +`cargo-audit` is not run as a second mandatory gate because `cargo-deny` already consumes the RustSec advisory database. Adding duplicate tooling without a distinct policy would increase noise rather than coverage. + +### Licenses + +The current allowlist is: + +- `Apache-2.0`; +- `MIT`; +- `Unicode-3.0`; +- `Unlicense`. + +The allowlist reflects licenses present in the locked dependency graph. Adding a license requires an explicit `deny.toml` change and review; CI must not silently broaden acceptance. + +### Bans and sources + +- Wildcard dependency versions are denied. +- Multiple versions are reported as warnings so they remain visible without blocking justified transitive duplication. +- Unknown registries and unknown Git sources are denied. +- crates.io is the only allowed registry. +- No Git dependency is allowlisted by default. + +## Compatibility checks + +### MSRV + +The workspace declares Rust 1.85.0 as its minimum supported Rust version. CI checks and tests the complete workspace with that toolchain. + +### Platform matrix + +CI retains stable-Rust tests on: + +- Ubuntu; +- Windows; +- macOS. + +### WebAssembly + +The core parser is compile-checked with: + +```bash +cargo check -p zstdscope --target wasm32-unknown-unknown --no-default-features --locked +``` + +This gate protects the dependency and platform-neutral parser boundary. It does not claim that JavaScript bindings or a browser UI already exist. + +### Semantic-version checks + +`cargo-semver-checks` was evaluated but is not a mandatory per-PR gate at this stage. ZstdScope is pre-1.0, and not every intentional public-model evolution should be blocked by an automated compatibility verdict. + +It should be run during release/API review when a release claims compatibility with an earlier published library version, and it should be reconsidered as a mandatory gate before 1.0. Intentional breaking changes still require documentation and normal review. + +## Packaging and distribution checks + +CI verifies both workspace packages with locked dependencies: + +```bash +cargo package -p zstdscope --list --locked +cargo publish -p zstdscope --dry-run --locked +cargo package -p zstdscope-cli --list --locked +cargo publish -p zstdscope-cli --dry-run --locked +``` + +The generated CLI package is installed from `target/package` into a temporary root. The resulting binary must report the package version and successfully inspect a minimal Standard Frame in text and JSON modes. + +Pull-request CI never receives crates.io publish credentials. Actual publication is a deliberate release operation after the exact commit and package contents have been reviewed. + +## Required quality gates + +The required workflow keeps the existing gates and adds the controls above: + +```bash +cargo fmt --check +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo test --workspace --all-features --locked +RUSTDOCFLAGS="-D warnings" cargo doc -p zstdscope --all-features --no-deps --locked +cargo test -p zstdscope --no-default-features --locked +``` + +A change is not complete merely because one operating system or one feature set passes. + +## Exceptions + +Security, license, source, or CI exceptions must be narrow and repository-visible. The change must explain: + +- the exact rule being relaxed; +- the affected package/action and version; +- the risk assessment; +- why a safer alternative is unavailable; +- how and when the exception will be removed. + +Permanent broad suppressions are not an acceptable substitute for resolving the underlying dependency or workflow issue. diff --git a/docs/adr/0005-versioned-cli-json.md b/docs/adr/0005-versioned-cli-json.md new file mode 100644 index 0000000..9c283ce --- /dev/null +++ b/docs/adr/0005-versioned-cli-json.md @@ -0,0 +1,92 @@ +# ADR 0005: Versioned CLI JSON DTO boundary + +- Status: Accepted +- Date: 2026-08-27 + +## Context + +The original `zstdscope inspect --json` implementation serialized the public Rust inspection model directly. Explicit Serde attributes and black-box tests made that representation predictable, but they did not separate two independently evolving contracts: + +```text +public Rust model ──► CLI JSON wire format +``` + +A Rust API refactor could therefore become an accidental JSON breaking change. The public model also contains `u64` offsets, lengths, sizes, and Frame Content Size values that cannot always be represented exactly by a JavaScript `Number`. + +ZstdScope needs an intentional machine-readable boundary before browser, editor, and scripting consumers become more numerous. + +## Decision + +### Dedicated private DTOs + +The CLI owns private `Json*V1` DTOs and maps the public `ZstdFile` model into them. The parser core does not own CLI JSON formatting. + +The core crate may continue to expose optional Serde support for consumers that deliberately serialize the Rust model. That representation is not the CLI JSON compatibility contract. + +### Version discriminator + +Every successful CLI JSON document includes: + +```json +{ + "schema_version": 1 +} +``` + +### Integer representation + +All values originating from Rust `u64` fields are encoded as decimal strings. This includes input size, byte-span offsets and lengths, derived window size, and Frame Content Size. + +Values whose source types are bounded `u8`, `u32`, or `usize` remain JSON numbers in schema version 1. + +### Structural fidelity + +The DTO must preserve inspector-specific distinctions: + +- absent Dictionary ID versus an explicitly encoded zero; +- RLE declared size versus its one-byte encoded content size; +- explicit Standard versus Skippable frame tags; +- source spans for encoded fields. + +### Compatibility policy + +Before 1.0, backward-compatible additive fields may be introduced without changing `schema_version` when existing consumers that ignore unknown fields remain valid. + +The following changes are breaking and require a new schema version plus release-note documentation: + +- removing or renaming a field; +- changing a field's JSON type; +- changing enum names, tags, or tagged-union shape; +- changing the meaning or units of an existing field; +- changing the decimal-string convention for `u64` values; +- erasing an inspector-specific distinction represented by schema version 1. + +The human-readable renderer is a separate presentation contract and must not change merely because DTO internals are refactored. + +## Consequences + +### Positive + +- Rust API evolution no longer changes CLI JSON automatically. +- JavaScript consumers can preserve every `u64` value exactly. +- Schema changes become deliberate, reviewable, and testable. +- The parser core remains independent from CLI wire-format policy. + +### Trade-offs + +- Mapping code duplicates the shape of selected public model fields. +- Decimal strings require numeric parsing for consumers that perform arithmetic. +- New public model fields are not exposed in CLI JSON until the DTO is updated intentionally. + +## Verification + +Black-box CLI tests cover: + +- a complete schema-version-1 fixture; +- Standard and Skippable frame tagged shapes; +- decimal-string `u64` values and `snake_case` names; +- absent versus explicitly encoded zero Dictionary IDs; +- RLE declared versus encoded sizes; +- non-zero failure with no partial JSON output. + +Any future schema version must add equivalent contract-focused tests before release. diff --git a/docs/adr/0006-cli-distribution.md b/docs/adr/0006-cli-distribution.md new file mode 100644 index 0000000..67f0ec6 --- /dev/null +++ b/docs/adr/0006-cli-distribution.md @@ -0,0 +1,95 @@ +# ADR 0006: Distribute the CLI as a crates.io package + +- Status: Accepted +- Date: 2026-08-27 + +## Context + +The reusable `zstdscope` library is published on crates.io, while the `zstdscope-cli` workspace package originally used `publish = false`. Users therefore needed a repository checkout to run the command. + +The evaluated channels were: + +1. publish the CLI crate and install it with Cargo; +2. publish prebuilt GitHub Release binaries; +3. support both. + +Prebuilt binaries would require a maintained target matrix, reproducible release automation, checksums, signing/provenance policy, and asset lifecycle management. Those controls are not yet present. + +## Decision + +### Primary release channel + +Publish `zstdscope-cli` on crates.io. The package name remains `zstdscope-cli`, while the installed executable remains `zstdscope`: + +```bash +cargo install zstdscope-cli --locked +``` + +Before the first CLI crate release is published, users may install the current repository revision without a manual checkout: + +```bash +cargo install --git https://github.com/tappe9/zstdscope zstdscope-cli --locked +``` + +### Package boundary + +The library and CLI remain separate crates: + +- `zstdscope` owns parsing, validation, and the public inspection model; +- `zstdscope-cli` owns filesystem input policy, CLI arguments, presentation, JSON DTOs, diagnostics, and process exit behavior. + +The CLI package depends on a released `zstdscope` version as well as the workspace path, allowing workspace development and a self-contained crates.io package. + +### Supported platforms + +The supported source-build matrix is the set continuously exercised by CI: + +- Ubuntu x86_64; +- Windows x86_64 using MSVC; +- macOS arm64. + +Other Rust-supported targets are best effort. This decision does not promise prebuilt artifacts for any target. + +### Prebuilt binaries + +GitHub Release binaries are deferred. They may be added by a later ADR only after automation defines: + +- supported target triples; +- reproducible build inputs; +- checksums for every downloadable asset; +- signing or provenance policy; +- release failure and revocation handling. + +## Consequences + +### Positive + +- Users get a conventional Rust installation path without manually cloning the repository. +- The release mechanism reuses Cargo's source-build and dependency-resolution model. +- The project avoids prematurely maintaining unsigned, manually assembled binaries. +- Library and CLI responsibilities remain separated. + +### Trade-offs + +- Users need a Rust toolchain and compile the CLI locally. +- Installation time is longer than downloading a prebuilt executable. +- The CLI crate must be published after its exact `zstdscope` dependency version exists on crates.io. + +## Verification + +CI performs all of the following from the package boundary: + +```bash +cargo package -p zstdscope-cli --list --locked +cargo publish -p zstdscope-cli --dry-run --locked +cargo package -p zstdscope-cli --locked +cargo install --path target/package/zstdscope-cli- --locked +``` + +The installed artifact must pass: + +- `zstdscope --version`; +- human-readable `zstdscope inspect`; +- `zstdscope inspect --json` with `schema_version` 1. + +Publishing credentials and the actual crates.io release remain an explicit release operation; pull-request CI never receives publish credentials. From aa0eaac8674ed316c3fa590ca808a3aa549c7de7 Mon Sep 17 00:00:00 2001 From: y_takino <36617106+tappe9@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:31:10 +0900 Subject: [PATCH 8/8] refactor: tighten package and license policy --- .github/workflows/ci.yml | 3 ++- deny.toml | 1 - docs/SUPPLY-CHAIN.md | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2beafa..05fa7ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,7 +69,8 @@ jobs: package_dir="$(find target/package -maxdepth 1 -type d -name 'zstdscope-cli-*' -print -quit)" test -n "$package_dir" - package_version="${package_dir##*-}" + package_name="${package_dir##*/}" + package_version="${package_name#zstdscope-cli-}" install_root="$RUNNER_TEMP/zstdscope-install" cargo install --path "$package_dir" --locked --root "$install_root" diff --git a/deny.toml b/deny.toml index 5e2470b..6793143 100644 --- a/deny.toml +++ b/deny.toml @@ -9,7 +9,6 @@ allow = [ "Apache-2.0", "MIT", "Unicode-3.0", - "Unlicense", ] confidence-threshold = 0.8 diff --git a/docs/SUPPLY-CHAIN.md b/docs/SUPPLY-CHAIN.md index 515a416..510c753 100644 --- a/docs/SUPPLY-CHAIN.md +++ b/docs/SUPPLY-CHAIN.md @@ -46,10 +46,9 @@ The current allowlist is: - `Apache-2.0`; - `MIT`; -- `Unicode-3.0`; -- `Unlicense`. +- `Unicode-3.0`. -The allowlist reflects licenses present in the locked dependency graph. Adding a license requires an explicit `deny.toml` change and review; CI must not silently broaden acceptance. +The allowlist reflects the minimum license set needed by the locked dependency graph. A dependency with an `Unlicense OR MIT` expression is accepted through its MIT option; `Unlicense` is not broadly allowlisted. Adding a license requires an explicit `deny.toml` change and review, and CI must not silently broaden acceptance. ### Bans and sources @@ -100,7 +99,7 @@ cargo package -p zstdscope-cli --list --locked cargo publish -p zstdscope-cli --dry-run --locked ``` -The generated CLI package is installed from `target/package` into a temporary root. The resulting binary must report the package version and successfully inspect a minimal Standard Frame in text and JSON modes. +The generated CLI package is installed from `target/package` into a temporary root. The resulting binary must report the generated package version and successfully inspect a minimal Standard Frame in text and JSON modes. Pull-request CI never receives crates.io publish credentials. Actual publication is a deliberate release operation after the exact commit and package contents have been reviewed.