diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53060296..764c2942 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,9 +55,9 @@ jobs: git config --global user.email "user@sysand.org" git config --global user.name "Test User" - name: Test Core - run: cargo test --locked --package sysand-core --verbose --features filesystem,js,python,alltests + run: cargo test --locked --package sysand-core --verbose --features filesystem,networking,js,python,alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd - name: Test CLI - run: cargo test --locked --package sysand --verbose --features alltests + run: cargo test --locked --package sysand --verbose --features alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd build: strategy: diff --git a/.vscode/settings.json b/.vscode/settings.json index 3546edc2..ebd05564 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,14 +1,21 @@ { "cSpell.words": [ - "sysand", - "sysml", "kerml", "kpar", - "metamodel", "mdbook", - "thiserror", - "reqwest", + "metamodel", + "Ppmd", "pubgrub", - "pycache" - ], + "pycache", + "pyerr", + "pyfunction", + "pymodule", + "reqwest", + "strs", + "sysand", + "sysml", + "thiserror", + "werr", + "wrapfs" + ] } \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ee7df208..162c4a0b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -115,8 +115,8 @@ Run tests for main Rust crates. This excludes language bindings, because they have their own test suites: ```sh -cargo test -p sysand-core -F filesystem,js,python,alltests -cargo test -p sysand -F alltests +cargo test -p sysand-core -F filesystem,networking,js,python,alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd +cargo test -p sysand -F alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd ``` Run tests for all crates and language bindings (requires bindings dependencies): diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 7ba3d24a..f6e3e90f 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -15,6 +15,12 @@ homepage.workspace = true name = "sysand" crate-type = ["cdylib"] +[features] +kpar-bzip2 = ["sysand-core/kpar-bzip2"] +kpar-zstd = ["sysand-core/kpar-zstd"] +kpar-xz = ["sysand-core/kpar-xz"] +kpar-ppmd = ["sysand-core/kpar-ppmd"] + [dependencies] sysand-core = { path = "../../core", features = ["std", "filesystem", "networking"] } camino.workspace = true diff --git a/bindings/java/java-test/src/test/java/com/sensmetry/sysand/BasicTest.java b/bindings/java/java-test/src/test/java/com/sensmetry/sysand/BasicTest.java index c720d423..a9951710 100644 --- a/bindings/java/java-test/src/test/java/com/sensmetry/sysand/BasicTest.java +++ b/bindings/java/java-test/src/test/java/com/sensmetry/sysand/BasicTest.java @@ -6,6 +6,8 @@ import org.junit.jupiter.api.Test; +import com.sensmetry.sysand.model.CompressionMethod; + import static org.junit.jupiter.api.Assertions.*; import java.util.regex.Pattern; @@ -108,6 +110,29 @@ public void testBasicInfo() { } } + @Test + public void testProjectBuild() { + try { + java.nio.file.Path tempDir = java.nio.file.Files.createTempDirectory("sysand-test-build"); + com.sensmetry.sysand.Sysand.init("test_basic_info", "1.2.3", "MIT", tempDir); + + com.sensmetry.sysand.model.InterchangeProject project = com.sensmetry.sysand.Sysand.infoPath(tempDir); + assertExpectedProject(project); + + java.net.URI fileUri = tempDir.toUri(); + com.sensmetry.sysand.model.InterchangeProject[] projects = com.sensmetry.sysand.Sysand.info(fileUri, + tempDir); + assertEquals(projects.length, 1); + assertExpectedProject(projects[0]); + + com.sensmetry.sysand.Sysand.buildProject(tempDir.resolve("sysand-test-build.kpar"), tempDir, CompressionMethod.DEFLATED); + } catch (java.io.IOException e) { + fail("Failed during temporary directory operations or Sysand.info: " + e.getMessage()); + } catch (com.sensmetry.sysand.exceptions.SysandException e) { + fail("Failed during temporary directory operations or Sysand.info: " + e.getMessage()); + } + } + @Test public void testHttpInfo() { // TODO: Find a good mock server so that we can test this. diff --git a/bindings/java/java/src/main/java/com/sensmetry/sysand/Sysand.java b/bindings/java/java/src/main/java/com/sensmetry/sysand/Sysand.java index 7c2c7179..8da7b520 100644 --- a/bindings/java/java/src/main/java/com/sensmetry/sysand/Sysand.java +++ b/bindings/java/java/src/main/java/com/sensmetry/sysand/Sysand.java @@ -152,7 +152,7 @@ public static com.sensmetry.sysand.model.InterchangeProject[] info(java.net.URI * @param outputPath The path to the output file. * @param projectPath The path to the project. */ - public static native void buildProject(String outputPath, String projectPath) + private static native void buildProject(String outputPath, String projectPath, String compression) throws com.sensmetry.sysand.exceptions.SysandException; /** @@ -162,9 +162,9 @@ public static native void buildProject(String outputPath, String projectPath) * @param outputPath The path to the output file. * @param projectPath The path to the project. */ - public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Path projectPath) + public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Path projectPath, com.sensmetry.sysand.model.CompressionMethod compression) throws com.sensmetry.sysand.exceptions.SysandException { - buildProject(outputPath.toString(), projectPath.toString()); + buildProject(outputPath.toString(), projectPath.toString(), compression.toString()); } /** @@ -174,7 +174,7 @@ public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Pat * @param outputPath The path to the output file. * @param workspacePath The path to the workspace. */ - public static native void buildWorkspace(String outputPath, String workspacePath) + private static native void buildWorkspace(String outputPath, String workspacePath, String compression) throws com.sensmetry.sysand.exceptions.SysandException; /** @@ -184,8 +184,8 @@ public static native void buildWorkspace(String outputPath, String workspacePath * @param outputPath The path to the output file. * @param workspacePath The path to the workspace. */ - public static void buildWorkspace(java.nio.file.Path outputPath, java.nio.file.Path workspacePath) + public static void buildWorkspace(java.nio.file.Path outputPath, java.nio.file.Path workspacePath, com.sensmetry.sysand.model.CompressionMethod compression) throws com.sensmetry.sysand.exceptions.SysandException { - buildWorkspace(outputPath.toString(), workspacePath.toString()); + buildWorkspace(outputPath.toString(), workspacePath.toString(), compression.toString()); } } diff --git a/bindings/java/java/src/main/java/com/sensmetry/sysand/model/CompressionMethod.java b/bindings/java/java/src/main/java/com/sensmetry/sysand/model/CompressionMethod.java new file mode 100644 index 00000000..9e211c14 --- /dev/null +++ b/bindings/java/java/src/main/java/com/sensmetry/sysand/model/CompressionMethod.java @@ -0,0 +1,16 @@ +package com.sensmetry.sysand.model; + +public enum CompressionMethod { + // Store the files as is + STORED, + // Compress the files using Deflate + DEFLATED, + /// Compress the files using BZIP2. Only available when sysand is compiled with feature kpar-bzip2 + BZIP2, + /// Compress the files using ZStandard. Only available when sysand is compiled with feature kpar-zstd + ZSTD, + /// Compress the files using XZ. Only available when sysand is compiled with feature kpar-xz + XZ, + /// Compress the files using PPMd. Only available when sysand is compiled with feature kpar-ppmd + PPMD, +} diff --git a/bindings/java/plugin/src/main/java/org/sysand/maven/SysandBuildKParMojo.java b/bindings/java/plugin/src/main/java/org/sysand/maven/SysandBuildKParMojo.java index f272cefe..8b22913c 100644 --- a/bindings/java/plugin/src/main/java/org/sysand/maven/SysandBuildKParMojo.java +++ b/bindings/java/plugin/src/main/java/org/sysand/maven/SysandBuildKParMojo.java @@ -4,12 +4,16 @@ package org.sysand.maven; +import java.nio.file.Paths; + import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; +import com.sensmetry.sysand.model.CompressionMethod; + @Mojo(name = "build-kpar", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = false) public class SysandBuildKParMojo extends AbstractMojo { @@ -37,6 +41,14 @@ public class SysandBuildKParMojo extends AbstractMojo { @Parameter(property = "sysand.outputPath", required = true) private String outputPath; + /** + * KPAR compression method. Can be configured as + * {@code ...} or + * via {@code -Dsysand.compressionMethod=...}. + */ + @Parameter(property = "sysand.compressionMethod", required = false) + private String compressionMethod; + @Override public void execute() throws MojoExecutionException { if (projectPath == null && workspacePath == null) { @@ -47,14 +59,16 @@ public void execute() throws MojoExecutionException { throw new MojoExecutionException("Parameter 'outputPath' must be provided and non-empty"); } + CompressionMethod compression = compressionMethod == null ? CompressionMethod.DEFLATED : CompressionMethod.valueOf(compressionMethod.toUpperCase()); + try { if (workspacePath == null) { - getLog().info("Invoking Sysand.buildProject on: " + projectPath + " to " + outputPath); - com.sensmetry.sysand.Sysand.buildProject(outputPath, projectPath); + getLog().info("Invoking Sysand.buildProject on: " + projectPath + " to " + outputPath + " with compression " + compressionMethod); + com.sensmetry.sysand.Sysand.buildProject(Paths.get(outputPath), Paths.get(projectPath), compression); getLog().info("Sysand.buildProject completed successfully."); } else { - getLog().info("Invoking Sysand.buildWorkspace on: " + workspacePath + " to " + outputPath); - com.sensmetry.sysand.Sysand.buildWorkspace(outputPath, workspacePath); + getLog().info("Invoking Sysand.buildWorkspace on: " + workspacePath + " to " + outputPath + " with compression " + compressionMethod); + com.sensmetry.sysand.Sysand.buildWorkspace(Paths.get(outputPath), Paths.get(workspacePath), compression); getLog().info("Sysand.buildWorkspace completed successfully."); } } catch (com.sensmetry.sysand.exceptions.SysandException e) { diff --git a/bindings/java/scripts/java-builder.py b/bindings/java/scripts/java-builder.py index 9366f50c..804c00aa 100755 --- a/bindings/java/scripts/java-builder.py +++ b/bindings/java/scripts/java-builder.py @@ -11,7 +11,7 @@ import platform import shutil import subprocess -from typing import Any +from typing import Any, Union ROOT_DIR = Path(__file__).absolute().parent.parent.parent.parent @@ -131,7 +131,7 @@ def compute_full_version(version: str, release_jar_version: bool) -> str: def build( use_release_build: bool, - use_existing_native_libs: Path | None, + use_existing_native_libs: Union[Path, None], sign_artifacts: bool, release_jar_version: bool, version: str, diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index 1464e170..69153f6b 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -11,7 +11,7 @@ use jni::{ }; use sysand_core::{ auth::Unauthenticated, - build::KParBuildError, + build::{KParBuildError, KparCompressionMethod}, commands, env::local_directory::{self, LocalWriteError}, info::InfoError, @@ -321,12 +321,26 @@ fn handle_build_error(env: &mut JNIEnv<'_>, error: KParBuildError } } +fn compression_from_java_string( + env: &mut JNIEnv<'_>, + compression: String, +) -> Option { + match KparCompressionMethod::try_from(compression) { + Ok(compression) => Some(compression), + Err(err) => { + env.throw_exception(ExceptionKind::SysandException, err.to_string()); + None + } + } +} + #[unsafe(no_mangle)] pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildProject<'local>( mut env: JNIEnv<'local>, _class: JClass<'local>, output_path: JString<'local>, project_path: JString<'local>, + compression: JString<'local>, ) { let Some(output_path) = env.get_str(&output_path, "outputPath") else { return; @@ -338,7 +352,14 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildProject<'local>( nominal_path: None, project_path: Utf8PathBuf::from(project_path), }; - let command_result = sysand_core::commands::build::do_build_kpar(&project, &output_path, true); + let Some(compression) = env.get_str(&compression, "compression") else { + return; + }; + let Some(compression) = compression_from_java_string(&mut env, compression) else { + return; + }; + let command_result = + sysand_core::commands::build::do_build_kpar(&project, &output_path, compression, true); match command_result { Ok(_) => {} Err(error) => handle_build_error(&mut env, error), @@ -351,6 +372,7 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildWorkspace<'local>( _class: JClass<'local>, output_path: JString<'local>, workspace_path: JString<'local>, + compression: JString<'local>, ) { let Some(output_path) = env.get_str(&output_path, "outputPath") else { return; @@ -365,6 +387,12 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildWorkspace<'local>( return; } }; + let Some(compression) = env.get_str(&compression, "compression") else { + return; + }; + let Some(compression) = compression_from_java_string(&mut env, compression) else { + return; + }; match wrapfs::create_dir_all(&output_path) { Ok(_) => {} Err(error) => { @@ -372,8 +400,13 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildWorkspace<'local>( return; } } - let command_result = - sysand_core::commands::build::do_build_workspace_kpars(&workspace, &output_path, true); + + let command_result = sysand_core::commands::build::do_build_workspace_kpars( + &workspace, + &output_path, + compression, + true, + ); match command_result { Ok(_) => {} Err(error) => handle_build_error(&mut env, error), diff --git a/bindings/js/src/env/local_storage.rs b/bindings/js/src/env/local_storage.rs index 424a7827..53169b3a 100644 --- a/bindings/js/src/env/local_storage.rs +++ b/bindings/js/src/env/local_storage.rs @@ -178,7 +178,7 @@ impl WriteEnvironment for LocalBrowserStorageEnvironment { .read_string(self.versions_path(&uri)) .map_err(Error::LocalStorage)?; - let mut kept_versions = "".to_string(); + let mut kept_versions = String::new(); let mut found = false; for current_version in current_versions.lines() { diff --git a/bindings/py/Cargo.toml b/bindings/py/Cargo.toml index 6e9886b1..6ec4b33e 100644 --- a/bindings/py/Cargo.toml +++ b/bindings/py/Cargo.toml @@ -15,6 +15,10 @@ homepage.workspace = true default = ["extension-module", "abi3"] extension-module = ["pyo3/extension-module"] abi3 = ["pyo3/abi3", "pyo3/abi3-py38"] +kpar-bzip2 = ["sysand-core/kpar-bzip2", "sysand/kpar-bzip2"] +kpar-zstd = ["sysand-core/kpar-zstd", "sysand/kpar-zstd"] +kpar-xz = ["sysand-core/kpar-xz", "sysand/kpar-xz"] +kpar-ppmd = ["sysand-core/kpar-ppmd", "sysand/kpar-ppmd"] [dependencies] sysand-core = { path = "../../core", features = ["python", "filesystem", "networking"] } diff --git a/bindings/py/python/sysand/__init__.py b/bindings/py/python/sysand/__init__.py index ecf42c76..b1897bc8 100644 --- a/bindings/py/python/sysand/__init__.py +++ b/bindings/py/python/sysand/__init__.py @@ -7,6 +7,7 @@ InterchangeProjectInfo, InterchangeProjectChecksum, InterchangeProjectMetadata, + CompressionMethod, ) from ._info import info_path, info @@ -47,6 +48,7 @@ "InterchangeProjectInfo", "InterchangeProjectChecksum", "InterchangeProjectMetadata", + "CompressionMethod", ## Add "add", ## Remove diff --git a/bindings/py/python/sysand/_build.py b/bindings/py/python/sysand/_build.py index 2397ad83..ed49cff3 100644 --- a/bindings/py/python/sysand/_build.py +++ b/bindings/py/python/sysand/_build.py @@ -1,14 +1,22 @@ from __future__ import annotations +from sysand._model import CompressionMethod import sysand._sysand_core as sysand_rs # type: ignore from pathlib import Path -def build(output_path: str | Path, project_path: str | Path | None = None) -> None: +def build( + output_path: str | Path, + project_path: str | Path | None = None, + compression: CompressionMethod | None = None, +) -> None: if project_path is not None: project_path = str(project_path) - sysand_rs.do_build_py(str(output_path), project_path) + + # comp = None if compression is None else _convert_compression(compression) + comp = None if compression is None else compression.name + sysand_rs.do_build_py(str(output_path), project_path, comp) __all__ = [ diff --git a/bindings/py/python/sysand/_model.py b/bindings/py/python/sysand/_model.py index 4e84cae6..87a6651c 100644 --- a/bindings/py/python/sysand/_model.py +++ b/bindings/py/python/sysand/_model.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT OR Apache-2.0 +from enum import Enum, auto import typing import datetime @@ -36,9 +37,25 @@ class InterchangeProjectMetadata(typing.TypedDict): checksum: typing.Optional[typing.List[InterchangeProjectChecksum]] +class CompressionMethod(Enum): + STORED = auto() + """Store the files as is""" + DEFLATED = auto() + """Compress the files using Deflate""" + BZIP2 = auto() + """Compress the files using BZIP2. Only available when sysand is compiled with feature kpar-bzip2""" + ZSTD = auto() + """Compress the files using ZStandard. Only available when sysand is compiled with feature kpar-zstd""" + XZ = auto() + """Compress the files using XZ. Only available when sysand is compiled with feature kpar-xz""" + PPMD = auto() + """Compress the files using PPMd. Only available when sysand is compiled with feature kpar-ppmd""" + + __all__ = [ "InterchangeProjectUsage", "InterchangeProjectInfo", "InterchangeProjectChecksum", "InterchangeProjectMetadata", + "CompressionMethod", ] diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index c1b70633..38b705e2 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -12,7 +12,7 @@ use semver::{Version, VersionReq}; use sysand_core::{ add::do_add, auth::Unauthenticated, - build::{KParBuildError, do_build_kpar}, + build::{KParBuildError, KparCompressionMethod, do_build_kpar}, commands::{ env::{EnvError, do_env_local_dir}, init::do_init_local_file, @@ -177,9 +177,13 @@ fn do_info_py( #[pyfunction(name = "do_build_py")] #[pyo3( - signature = (output_path, project_path), + signature = (output_path, project_path, compression), )] -fn do_build_py(output_path: String, project_path: Option) -> PyResult<()> { +fn do_build_py( + output_path: String, + project_path: Option, + compression: Option, +) -> PyResult<()> { let _ = pyo3_log::try_init(); let Some(current_project_path) = project_path else { @@ -190,7 +194,15 @@ fn do_build_py(output_path: String, project_path: Option) -> PyResult<() project_path: current_project_path.into(), }; - do_build_kpar(&project, &output_path, true) + let compression = match compression { + Some(compression) => match KparCompressionMethod::try_from(compression) { + Ok(compression) => compression, + Err(err) => return Err(PyValueError::new_err(err.to_string())), + }, + None => KparCompressionMethod::default(), + }; + + do_build_kpar(&project, &output_path, compression, true) .map(|_| ()) .map_err(|err| match err { KParBuildError::ProjectRead(_) => PyRuntimeError::new_err(err.to_string()), @@ -571,6 +583,8 @@ pub fn sysand_py(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(do_include_py, m)?)?; m.add_function(wrap_pyfunction!(do_exclude_py, m)?)?; m.add_function(wrap_pyfunction!(do_env_install_path_py, m)?)?; + // Currently this interop is done with strings instead + // m.add_class::()?; m.add("DEFAULT_ENV_NAME", DEFAULT_ENV_NAME)?; Ok(()) diff --git a/bindings/py/tests/test_basic.py b/bindings/py/tests/test_basic.py index 7f0eb8e7..50a8009a 100644 --- a/bindings/py/tests/test_basic.py +++ b/bindings/py/tests/test_basic.py @@ -3,7 +3,7 @@ from pathlib import Path import re import os -from typing import List +from typing import List, Union import pytest from pytest_httpserver import HTTPServer @@ -170,7 +170,10 @@ def test_index_info(caplog: pytest.LogCaptureFixture, httpserver: HTTPServer) -> assert meta["checksum"] is None -def compare_sources(sources: List[str], expected_sources: List[str]) -> None: +def compare_sources( + sources: Union[List[Path], List[str]], + expected_sources: Union[List[Path], List[str]], +) -> None: assert len(sources) == len(expected_sources) for source, expected_source in zip(sources, expected_sources): assert os.path.samefile(source, expected_source), ( @@ -178,7 +181,7 @@ def compare_sources(sources: List[str], expected_sources: List[str]) -> None: ) -def test_end_to_end_install_sources(): +def test_end_to_end_install_sources() -> None: with tempfile.TemporaryDirectory() as tmp_main: with tempfile.TemporaryDirectory() as tmp_dep: tmp_main = Path(tmp_main).resolve() @@ -242,3 +245,24 @@ def test_end_to_end_install_sources(): ), ], ) + + +@pytest.mark.parametrize( + "compression", + [None, sysand.CompressionMethod.STORED, sysand.CompressionMethod.DEFLATED], +) +def test_build(compression: Union[sysand.CompressionMethod, None]) -> None: + with tempfile.TemporaryDirectory() as tmp_main: + tmp_main = Path(tmp_main).resolve() + sysand.new("test_build", "1.2.3", tmp_main) + + with open(tmp_main / "src.sysml", "w") as f: + f.write("package Src;") + + sysand.include(tmp_main, "src.sysml") + + sysand.build( + output_path=tmp_main / "test_build.kpar", + project_path=tmp_main, + compression=compression, + ) diff --git a/core/Cargo.toml b/core/Cargo.toml index 9059072c..129f1e4c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -22,6 +22,11 @@ python = ["dep:pyo3"] js = ["dep:wasm-bindgen"] filesystem = ["dep:camino-tempfile", "dep:dirs", "dep:zip"] networking = ["dep:reqwest", "dep:gix"] # "dep:reqwest-middleware", "dep:partialzip" +# Different compression methods for creating KPARs +kpar-bzip2 = ["zip?/bzip2"] +kpar-zstd = ["zip?/zstd"] +kpar-xz = ["zip?/xz"] +kpar-ppmd = ["zip?/ppmd"] alltests = [] [dependencies] @@ -53,7 +58,7 @@ typed-path = { version = "0.12.3", default-features = false } walkdir = "2.5.0" # unicode-normalization = { version = "0.1.24", default-features = false } wasm-bindgen = { version = "0.2.114", default-features = false, optional = true } -zip = { version = "7.2.0", default-features = false, optional = true } +zip = { version = "7.2.0", default-features = false, optional = true, features = ["deflate"] } url = { version = "2.5.8", default-features = false } gix = { version = "0.80.0", default-features = false, optional = true, features = ["blocking-http-transport-reqwest", "blocking-network-client", "worktree-mutation"] } logos = "0.16.1" diff --git a/core/src/commands/build.rs b/core/src/commands/build.rs index 181f7e6b..c4d16d80 100644 --- a/core/src/commands/build.rs +++ b/core/src/commands/build.rs @@ -1,22 +1,121 @@ -#[cfg(feature = "filesystem")] use camino::Utf8Path; use thiserror::Error; use crate::{ env::utils::{CloneError, ErrorBound}, + include::IncludeError, model::InterchangeProjectValidationError, project::{ ProjectRead, local_kpar::{IntoKparError, LocalKParProject}, - local_src::LocalSrcError, + local_src::{LocalSrcError, LocalSrcProject}, utils::{FsIoError, ZipArchiveError}, }, - workspace::WorkspaceReadError, + workspace::{Workspace, WorkspaceReadError}, }; -#[cfg(feature = "filesystem")] -use crate::{project::local_src::LocalSrcProject, workspace::Workspace}; -use super::include::IncludeError; +#[derive(Default, Copy, Clone, Debug, PartialEq, Eq)] +// Currently python interop is done with strings instead +// in part to have less boilerplate, in part because the old +// Python we use doesn't have pattern matching which ensures +// all cases are covered +// #[cfg_attr(feature = "python", pyclass(eq))] +pub enum KparCompressionMethod { + /// Store the files as is + Stored, + /// Compress the files using Deflate + #[default] + Deflated, + /// Compress the files using BZIP2 + #[cfg(feature = "kpar-bzip2")] + Bzip2, + /// Compress the files using ZStandard + #[cfg(feature = "kpar-zstd")] + Zstd, + /// Compress the files using XZ + #[cfg(feature = "kpar-xz")] + Xz, + /// Compress the files using PPMd + #[cfg(feature = "kpar-ppmd")] + Ppmd, +} + +impl From for zip::CompressionMethod { + fn from(value: KparCompressionMethod) -> Self { + match value { + KparCompressionMethod::Stored => zip::CompressionMethod::Stored, + KparCompressionMethod::Deflated => zip::CompressionMethod::Deflated, + #[cfg(feature = "kpar-bzip2")] + KparCompressionMethod::Bzip2 => zip::CompressionMethod::Bzip2, + #[cfg(feature = "kpar-zstd")] + KparCompressionMethod::Zstd => zip::CompressionMethod::Zstd, + #[cfg(feature = "kpar-xz")] + KparCompressionMethod::Xz => zip::CompressionMethod::Xz, + #[cfg(feature = "kpar-ppmd")] + KparCompressionMethod::Ppmd => zip::CompressionMethod::Ppmd, + } + } +} + +#[derive(Debug, Error)] +pub enum CompressionMethodParseError { + #[error("Compile sysand with feature {feature} to use {compression} compression")] + SuggestFeature { + compression: String, + feature: String, + }, + #[error("{0}")] + Invalid(String), +} + +impl TryFrom for KparCompressionMethod { + type Error = CompressionMethodParseError; + + fn try_from(value: String) -> Result { + Self::try_from(value.as_str()) + } +} + +impl TryFrom<&str> for KparCompressionMethod { + type Error = CompressionMethodParseError; + fn try_from(value: &str) -> Result { + match value { + "STORED" => Ok(KparCompressionMethod::Stored), + "DEFLATED" => Ok(KparCompressionMethod::Deflated), + #[cfg(feature = "kpar-bzip2")] + "BZIP2" => Ok(KparCompressionMethod::Bzip2), + #[cfg(not(feature = "kpar-bzip2"))] + "BZIP2" => Err(CompressionMethodParseError::SuggestFeature { + compression: value.into(), + feature: "kpar-bzip2".into(), + }), + #[cfg(feature = "kpar-zstd")] + "ZSTD" => Ok(KparCompressionMethod::Zstd), + #[cfg(not(feature = "kpar-zstd"))] + "ZSTD" => Err(CompressionMethodParseError::SuggestFeature { + compression: value.into(), + feature: "kpar-zstd".into(), + }), + #[cfg(feature = "kpar-xz")] + "XZ" => Ok(KparCompressionMethod::Xz), + #[cfg(not(feature = "kpar-xz"))] + "XZ" => Err(CompressionMethodParseError::SuggestFeature { + compression: value.into(), + feature: "kpar-xz".into(), + }), + #[cfg(feature = "kpar-ppmd")] + "PPMD" => Ok(KparCompressionMethod::Ppmd), + #[cfg(not(feature = "kpar-ppmd"))] + "PPMD" => Err(CompressionMethodParseError::SuggestFeature { + compression: value.into(), + feature: "kpar-ppmd".into(), + }), + _ => Err(CompressionMethodParseError::Invalid(format!( + "Compression method `{value}` is invalid" + ))), + } + } +} #[derive(Error, Debug)] pub enum KParBuildError { @@ -97,7 +196,6 @@ impl From> } } -#[cfg(feature = "filesystem")] pub fn default_kpar_file_name( project: &Pr, ) -> Result> { @@ -115,10 +213,10 @@ pub fn default_kpar_file_name( )) } -#[cfg(feature = "filesystem")] pub fn do_build_kpar, Pr: ProjectRead>( project: &Pr, path: P, + compression: KparCompressionMethod, canonicalise: bool, ) -> Result> { use crate::project::local_src::LocalSrcProject; @@ -152,13 +250,17 @@ pub fn do_build_kpar, Pr: ProjectRead>( } } - Ok(LocalKParProject::from_project(&local_project, path)?) + Ok(LocalKParProject::from_project( + &local_project, + path, + compression.into(), + )?) } -#[cfg(feature = "filesystem")] pub fn do_build_workspace_kpars>( workspace: &Workspace, path: P, + compression: KparCompressionMethod, canonicalise: bool, ) -> Result, KParBuildError> { let mut result = Vec::new(); @@ -169,7 +271,7 @@ pub fn do_build_workspace_kpars>( }; let file_name = default_kpar_file_name(&project)?; let output_path = path.as_ref().join(file_name); - let kpar_project = do_build_kpar(&project, &output_path, canonicalise)?; + let kpar_project = do_build_kpar(&project, &output_path, compression, canonicalise)?; result.push(kpar_project); } Ok(result) diff --git a/core/src/project/gix_git_download.rs b/core/src/project/gix_git_download.rs index 6a5dc2a1..a7c54238 100644 --- a/core/src/project/gix_git_download.rs +++ b/core/src/project/gix_git_download.rs @@ -226,10 +226,12 @@ mod tests { // sleep(Duration::from_millis(100)); - let project = GixDownloadedProject::new(format!( - "file://{}", - repo_dir.path().canonicalize()?.display() - ))?; + let canonical = repo_dir.path().canonicalize()?; + // On Windows, canonicalize() returns extended-length paths with a `\\?\` + // prefix that gix cannot parse as a valid file URL. Strip it. + let path = canonical.to_str().unwrap(); + let path = path.strip_prefix(r"\\?\").unwrap_or(path); + let project = GixDownloadedProject::new(format!("file://{path}"))?; let (Some(info), Some(meta)) = project.get_project()? else { panic!("expected info and meta"); @@ -238,7 +240,7 @@ mod tests { assert_eq!(info.name, "basic_gix_access"); assert_eq!(meta.created, "123"); - let mut buf = "".to_string(); + let mut buf = String::new(); project .read_source("test.sysml")? .read_to_string(&mut buf)?; diff --git a/core/src/project/local_kpar.rs b/core/src/project/local_kpar.rs index 3705068f..50d2aff8 100644 --- a/core/src/project/local_kpar.rs +++ b/core/src/project/local_kpar.rs @@ -199,12 +199,12 @@ impl LocalKParProject { pub fn from_project>( from: &Pr, path: P, + compression: zip::CompressionMethod, ) -> Result> { let file = wrapfs::File::create(&path)?; let mut zip = zip::ZipWriter::new(file); - let options = zip::write::SimpleFileOptions::default() - .compression_method(zip::CompressionMethod::Stored); + let options = zip::write::SimpleFileOptions::default().compression_method(compression); let (info, meta) = from.get_project().map_err(IntoKparError::ProjectRead)?; let info = info.ok_or(IntoKparError::MissingInfo)?; @@ -403,7 +403,7 @@ mod tests { assert_eq!(info.version, "1.2.3"); assert_eq!(meta.created, "123"); - let mut src = "".to_string(); + let mut src = String::new(); project .read_source("test.sysml")? .read_to_string(&mut src)?; @@ -446,7 +446,7 @@ mod tests { assert_eq!(info.version, "1.2.3"); assert_eq!(meta.created, "123"); - let mut src = "".to_string(); + let mut src = String::new(); project .read_source("test.sysml")? .read_to_string(&mut src)?; diff --git a/core/src/project/local_src.rs b/core/src/project/local_src.rs index 1b8150c5..eefee79a 100644 --- a/core/src/project/local_src.rs +++ b/core/src/project/local_src.rs @@ -24,7 +24,7 @@ use crate::{ use super::utils::{FsIoError, ProjectDeserializationError, ProjectSerializationError}; -/// Project stored in a local directory as an extracted kpar archive. +/// Project stored in a local directory as an extracted KPAR archive. /// Source file paths with (unix) segments `segment1/.../segmentN` are /// re-interpreted as filesystem-native paths relative to `project_path`. #[derive(Clone, Debug)] diff --git a/core/src/project/reqwest_kpar_download.rs b/core/src/project/reqwest_kpar_download.rs index 4c10a1f9..161566ab 100644 --- a/core/src/project/reqwest_kpar_download.rs +++ b/core/src/project/reqwest_kpar_download.rs @@ -244,7 +244,7 @@ mod tests { assert_eq!(info.name, "test_basic_download_request"); assert_eq!(meta.created, "123"); - let mut src = "".to_string(); + let mut src = String::new(); project .read_source("test.sysml")? .read_to_string(&mut src)?; diff --git a/core/src/project/reqwest_src.rs b/core/src/project/reqwest_src.rs index c5da9cdd..3787635b 100644 --- a/core/src/project/reqwest_src.rs +++ b/core/src/project/reqwest_src.rs @@ -298,7 +298,7 @@ mod tests { assert_eq!(info.name, "test_basic_project_urls"); assert_eq!(meta.created, "0000-00-00T00:00:00.123456789Z"); - let mut src_buf = "".to_string(); + let mut src_buf = String::new(); project .read_source(Utf8UnixPath::new("Mekanïk/Kommandöh.sysml").to_path_buf())? .read_to_string(&mut src_buf)?; diff --git a/core/src/resolve/reqwest_http.rs b/core/src/resolve/reqwest_http.rs index 3049de4a..52f6b8ae 100644 --- a/core/src/resolve/reqwest_http.rs +++ b/core/src/resolve/reqwest_http.rs @@ -242,7 +242,7 @@ impl Iterator for HTTPProjects { } /// Tries treat IRIs as HTTP URLs, pointing either to source files stored remotely -/// or a KPar archive stored remotely. +/// or a KPAR archive stored remotely. /// /// If `prefer_ranged` is true, it attempts to poke the remote server to see if it /// appears to support HTTP Range requests. If successful, it uses `HTTPKparProjectRanged` diff --git a/core/tests/filesystem_env.rs b/core/tests/filesystem_env.rs index 4d9ad276..91b0b2e3 100644 --- a/core/tests/filesystem_env.rs +++ b/core/tests/filesystem_env.rs @@ -138,7 +138,7 @@ mod filesystem_tests { assert_eq!(read_info, Some(info.clone())); assert_eq!(read_meta, Some(meta.clone())); - let mut read_source_code = "".to_string(); + let mut read_source_code = String::new(); target_project .read_source(source_path)? diff --git a/core/tests/memory_env.rs b/core/tests/memory_env.rs index 78468977..ca52d6c5 100644 --- a/core/tests/memory_env.rs +++ b/core/tests/memory_env.rs @@ -86,7 +86,7 @@ fn env_manual_install() -> Result<(), Box> { assert_eq!(target_project.info, Some(info.clone())); assert_eq!(target_project.meta, Some(meta.clone())); - let mut read_source_code = "".to_string(); + let mut read_source_code = String::new(); target_project .read_source(source_path)? diff --git a/docs/src/commands/build.md b/docs/src/commands/build.md index 3dc9f5f7..33133187 100644 --- a/docs/src/commands/build.md +++ b/docs/src/commands/build.md @@ -26,4 +26,13 @@ if none is found uses the current directory instead. `/output/-.kpar` depending on whether the current project belongs to a workspace or not). +## Options + +- `-c`, `--compression`: Method to compress the files in the KPAR. + Possible values: + - `stored`: Store the files as is + - `deflated`: Compress the files using Deflate + + [default: `deflated`] + {{#include ./partials/global_opts.md}} diff --git a/sysand/Cargo.toml b/sysand/Cargo.toml index 0c1f6479..2fab36d0 100644 --- a/sysand/Cargo.toml +++ b/sysand/Cargo.toml @@ -15,6 +15,11 @@ homepage.workspace = true default = ["std"] std = ["anyhow/std", "clap/std", "fluent-uri/std", "log/std", "toml/std", "typed-path/std"] alltests = [] +# Different compression methods for creating KPARs +kpar-bzip2 = ["sysand-core/kpar-bzip2"] +kpar-zstd = ["sysand-core/kpar-zstd"] +kpar-xz = ["sysand-core/kpar-xz"] +kpar-ppmd = ["sysand-core/kpar-ppmd"] [dependencies] # General @@ -34,8 +39,6 @@ spdx = "0.13.4" fluent-uri = { version = "0.4.1", default-features = false } typed-path = { version = "0.12.3", default-features = false } url = { version = "2.5.8", default-features = false } -# Enables a bunch of additional compression/encryption features not enabled by default in sysand-core -zip = { version = "7.2.0" } pubgrub = { version = "0.3.0", default-features = false } indexmap = "2.13.0" tokio = { version = "1.50.0", default-features = false } diff --git a/sysand/src/cli.rs b/sysand/src/cli.rs index 792bf656..e6281a1a 100644 --- a/sysand/src/cli.rs +++ b/sysand/src/cli.rs @@ -11,6 +11,7 @@ use camino::Utf8PathBuf; use clap::{ValueEnum, builder::StyledStr, crate_authors}; use fluent_uri::Iri; use semver::VersionReq; +use sysand_core::build::KparCompressionMethod; use crate::env_vars; @@ -161,6 +162,10 @@ pub enum Command { /// on whether the current project belongs to a workspace or not). #[clap(verbatim_doc_comment)] path: Option, + #[clap(verbatim_doc_comment)] + /// Method to compress the files in the KPAR + #[arg(short = 'c', long, default_value_t, value_enum)] + compression: KparCompressionMethodCli, }, /// Create or update lockfile Lock { @@ -254,6 +259,64 @@ pub struct ProjectLocatorArgs { pub path: Option, } +#[derive(clap::ValueEnum, Default, Copy, Clone, Debug)] +#[clap(rename_all = "lowercase")] +pub enum KparCompressionMethodCli { + /// Store the files as is + Stored, + /// Compress the files using Deflate + #[default] + Deflated, + /// Compress the files using BZIP2 + #[cfg(feature = "kpar-bzip2")] + Bzip2, + /// Compress the files using ZStandard + #[cfg(feature = "kpar-zstd")] + Zstd, + /// Compress the files using XZ + #[cfg(feature = "kpar-xz")] + Xz, + /// Compress the files using PPMd + #[cfg(feature = "kpar-ppmd")] + Ppmd, +} + +impl From for KparCompressionMethod { + fn from(value: KparCompressionMethodCli) -> Self { + match value { + KparCompressionMethodCli::Stored => KparCompressionMethod::Stored, + KparCompressionMethodCli::Deflated => KparCompressionMethod::Deflated, + #[cfg(feature = "kpar-bzip2")] + KparCompressionMethodCli::Bzip2 => KparCompressionMethod::Bzip2, + #[cfg(feature = "kpar-zstd")] + KparCompressionMethodCli::Zstd => KparCompressionMethod::Zstd, + #[cfg(feature = "kpar-xz")] + KparCompressionMethodCli::Xz => KparCompressionMethod::Xz, + #[cfg(feature = "kpar-ppmd")] + KparCompressionMethodCli::Ppmd => KparCompressionMethod::Ppmd, + } + } +} + +// This is implemented mainly so that if KparCompressionMethod gets a new member +// and KparCompressionMethodCli isn't updated it would give a compilation error +impl From for KparCompressionMethodCli { + fn from(value: KparCompressionMethod) -> Self { + match value { + KparCompressionMethod::Stored => KparCompressionMethodCli::Stored, + KparCompressionMethod::Deflated => KparCompressionMethodCli::Deflated, + #[cfg(feature = "kpar-bzip2")] + KparCompressionMethod::Bzip2 => KparCompressionMethodCli::Bzip2, + #[cfg(feature = "kpar-zstd")] + KparCompressionMethod::Zstd => KparCompressionMethodCli::Zstd, + #[cfg(feature = "kpar-xz")] + KparCompressionMethod::Xz => KparCompressionMethodCli::Xz, + #[cfg(feature = "kpar-ppmd")] + KparCompressionMethod::Ppmd => KparCompressionMethodCli::Ppmd, + } + } +} + #[derive(Clone, Debug)] struct InvalidCommand { message: String, diff --git a/sysand/src/commands/build.rs b/sysand/src/commands/build.rs index 65ca1680..9e2c17a7 100644 --- a/sysand/src/commands/build.rs +++ b/sysand/src/commands/build.rs @@ -4,22 +4,24 @@ use anyhow::Result; use camino::Utf8Path; use sysand_core::{ - build::{do_build_kpar, do_build_workspace_kpars}, + build::{KparCompressionMethod, do_build_kpar, do_build_workspace_kpars}, project::local_src::LocalSrcProject, workspace::Workspace, }; pub fn command_build_for_project>( path: P, + compression: KparCompressionMethod, current_project: LocalSrcProject, ) -> Result<()> { - do_build_kpar(¤t_project, &path, true)?; + do_build_kpar(¤t_project, &path, compression, true)?; Ok(()) } pub fn command_build_for_workspace>( path: P, + compression: KparCompressionMethod, workspace: Workspace, ) -> Result<()> { log::warn!( @@ -28,7 +30,7 @@ pub fn command_build_for_workspace>( releases. For the status of this feature, see\n\ https://github.com/sensmetry/sysand/issues/101." ); - do_build_workspace_kpars(&workspace, &path, true)?; + do_build_workspace_kpars(&workspace, &path, compression, true)?; Ok(()) } diff --git a/sysand/src/lib.rs b/sysand/src/lib.rs index dbcf2e5c..41bd7b28 100644 --- a/sysand/src/lib.rs +++ b/sysand/src/lib.rs @@ -607,7 +607,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { no_index_symbols, } => command_include(paths, add_checksum, !no_index_symbols, current_project), cli::Command::Exclude { paths } => command_exclude(paths, current_project), - cli::Command::Build { path } => { + cli::Command::Build { path, compression } => { if let Some(current_project) = current_project { // Even if we are in a workspace, the project takes precedence. let path = if let Some(path) = path { @@ -625,7 +625,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { output_dir.push(name); output_dir }; - command_build_for_project(path, current_project) + command_build_for_project(path, compression.into(), current_project) } else { // If the workspace is also missing, report an error about // missing project because that is what the user is more likely @@ -637,7 +637,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { if !wrapfs::is_dir(&output_dir)? { wrapfs::create_dir(&output_dir)?; } - command_build_for_workspace(output_dir, current_workspace) + command_build_for_workspace(output_dir, compression.into(), current_workspace) } } cli::Command::Sources { sources_opts } => { diff --git a/sysand/tests/cli_build.rs b/sysand/tests/cli_build.rs index 180f3f93..f2d5aa77 100644 --- a/sysand/tests/cli_build.rs +++ b/sysand/tests/cli_build.rs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use assert_cmd::prelude::*; +use clap::ValueEnum; use predicates::prelude::*; +use std::io::{Read, Write}; +use sysand::cli::KparCompressionMethodCli; use sysand_core::{ model::{InterchangeProjectChecksumRaw, KerMlChecksumAlg}, project::{ProjectRead, local_kpar::LocalKParProject}, @@ -146,3 +149,71 @@ fn test_workspace_build() -> Result<(), Box> { Ok(()) } + +#[test] +fn test_compression_methods() -> Result<(), Box> { + let compressions = KparCompressionMethodCli::value_variants(); + test_compression_method(None)?; + for compression in compressions { + test_compression_method(Some(compression.to_possible_value().unwrap().get_name()))?; + } + Ok(()) +} + +fn test_compression_method(compression: Option<&str>) -> Result<(), Box> { + let (_temp_dir, cwd, out) = + run_sysand(["init", "--version", "1.2.3", "--name", "test_build"], None)?; + + { + let mut sysml_file = std::fs::File::create(cwd.join("test.sysml"))?; + sysml_file.write_all(b"package P;\n")?; + } + + out.assert().success(); + + let out = run_sysand_in(&cwd, ["include", "--no-index-symbols", "test.sysml"], None)?; + + out.assert().success(); + + let out = match compression { + Some(compression) => run_sysand_in( + &cwd, + ["build", "--compression", compression, "./test_build.kpar"], + None, + )?, + None => run_sysand_in(&cwd, ["build", "./test_build.kpar"], None)?, + }; + + out.assert().success(); + + let out = run_sysand_in( + &cwd, + ["info", "--path", cwd.join("test_build.kpar").as_str()], + None, + )?; + + out.assert() + .success() + .stdout(predicate::str::contains("Name: test_build")) + .stdout(predicate::str::contains("Version: 1.2.3")); + + let kpar_project = LocalKParProject::new_guess_root(cwd.join("test_build.kpar"))?; + + let (Some(info), Some(meta)) = kpar_project.get_project()? else { + panic!("failed to get built project info/meta"); + }; + + assert_eq!(info.name, "test_build"); + assert_eq!(info.version, "1.2.3"); + + assert_eq!(meta.checksum.as_ref().unwrap().len(), 1); + assert_eq!(meta.index.len(), 1); + assert_eq!(meta.index.get("P").unwrap(), "test.sysml"); + let mut src = String::new(); + kpar_project + .read_source("test.sysml")? + .read_to_string(&mut src)?; + + assert_eq!(src, "package P;\n"); + Ok(()) +} diff --git a/sysand/tests/cli_info.rs b/sysand/tests/cli_info.rs index 1bb68677..71134ebd 100644 --- a/sysand/tests/cli_info.rs +++ b/sysand/tests/cli_info.rs @@ -1391,7 +1391,7 @@ fn info_detailed_verbs() -> Result<(), Box> { if expected { out.assert().success(); let skipped = index.parse::()? - 1; - let mut expected_output = "".to_string(); + let mut expected_output = String::new(); for (i, line) in before.lines().enumerate() { if i != skipped { expected_output.push_str(line);