feat: add KPAR compression options

.github/workflows/test.yml

@@ -55,9 +55,9 @@ jobs:
           git config --global user.email "user@sysand.org"
           git config --global user.name "Test User"
       - name: Test Core
-        run: cargo test --locked --package sysand-core --verbose --features filesystem,js,python,alltests
+        run: cargo test --locked --package sysand-core --verbose --features filesystem,networking,js,python,alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd
       - name: Test CLI
-        run: cargo test --locked --package sysand --verbose --features alltests
+        run: cargo test --locked --package sysand --verbose --features alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd
 
   build:
     strategy:

.vscode/settings.json

@@ -1,14 +1,21 @@
 {
     "cSpell.words": [
-        "sysand",
-        "sysml",
         "kerml",
         "kpar",
-        "metamodel",
         "mdbook",
-        "thiserror",
-        "reqwest",
+        "metamodel",
+        "Ppmd",
         "pubgrub",
-        "pycache"
-    ],
+        "pycache",
+        "pyerr",
+        "pyfunction",
+        "pymodule",
+        "reqwest",
+        "strs",
+        "sysand",
+        "sysml",
+        "thiserror",
+        "werr",
+        "wrapfs"
+    ]
 }

DEVELOPMENT.md

@@ -115,8 +115,8 @@ Run tests for main Rust crates. This excludes language bindings, because they
 have their own test suites:
 
 ```sh
-cargo test -p sysand-core -F filesystem,js,python,alltests
-cargo test -p sysand -F alltests
+cargo test -p sysand-core -F filesystem,networking,js,python,alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd
+cargo test -p sysand -F alltests,kpar-bzip2,kpar-zstd,kpar-xz,kpar-ppmd
 ```
 
 Run tests for all crates and language bindings (requires bindings dependencies):

bindings/java/Cargo.toml

@@ -15,6 +15,12 @@ homepage.workspace = true
 name = "sysand"
 crate-type = ["cdylib"]
 
+[features]
+kpar-bzip2 = ["sysand-core/kpar-bzip2"]
+kpar-zstd = ["sysand-core/kpar-zstd"]
+kpar-xz = ["sysand-core/kpar-xz"]
+kpar-ppmd = ["sysand-core/kpar-ppmd"]
+
 [dependencies]
 sysand-core = { path = "../../core", features = ["std", "filesystem", "networking"] }
 camino.workspace = true

bindings/java/java-test/src/test/java/com/sensmetry/sysand/BasicTest.java

@@ -6,6 +6,8 @@
 
 import org.junit.jupiter.api.Test;
 
+import com.sensmetry.sysand.model.CompressionMethod;
+
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.util.regex.Pattern;
@@ -108,6 +110,29 @@ public void testBasicInfo() {
         }
     }
 
+    @Test
+    public void testProjectBuild() {
+        try {
+            java.nio.file.Path tempDir = java.nio.file.Files.createTempDirectory("sysand-test-build");
+            com.sensmetry.sysand.Sysand.init("test_basic_info", "1.2.3", "MIT", tempDir);
+
+            com.sensmetry.sysand.model.InterchangeProject project = com.sensmetry.sysand.Sysand.infoPath(tempDir);
+            assertExpectedProject(project);
+
+            java.net.URI fileUri = tempDir.toUri();
+            com.sensmetry.sysand.model.InterchangeProject[] projects = com.sensmetry.sysand.Sysand.info(fileUri,
+                    tempDir);
+            assertEquals(projects.length, 1);
+            assertExpectedProject(projects[0]);
+
+            com.sensmetry.sysand.Sysand.buildProject(tempDir.resolve("sysand-test-build.kpar"), tempDir, CompressionMethod.DEFLATED);
+        } catch (java.io.IOException e) {
+            fail("Failed during temporary directory operations or Sysand.info: " + e.getMessage());
+        } catch (com.sensmetry.sysand.exceptions.SysandException e) {
+            fail("Failed during temporary directory operations or Sysand.info: " + e.getMessage());
+        }
+    }
+
     @Test
     public void testHttpInfo() {
         // TODO: Find a good mock server so that we can test this.

bindings/java/java/src/main/java/com/sensmetry/sysand/Sysand.java

@@ -152,7 +152,7 @@ public static com.sensmetry.sysand.model.InterchangeProject[] info(java.net.URI
      * @param outputPath  The path to the output file.
      * @param projectPath The path to the project.
      */
-    public static native void buildProject(String outputPath, String projectPath)
+    private static native void buildProject(String outputPath, String projectPath, String compression)
             throws com.sensmetry.sysand.exceptions.SysandException;
 
     /**
@@ -162,9 +162,9 @@ public static native void buildProject(String outputPath, String projectPath)
      * @param outputPath  The path to the output file.
      * @param projectPath The path to the project.
      */
-    public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Path projectPath)
+    public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Path projectPath, com.sensmetry.sysand.model.CompressionMethod compression)
             throws com.sensmetry.sysand.exceptions.SysandException {
-        buildProject(outputPath.toString(), projectPath.toString());
+        buildProject(outputPath.toString(), projectPath.toString(), compression.toString());
     }
 
     /**
@@ -174,7 +174,7 @@ public static void buildProject(java.nio.file.Path outputPath, java.nio.file.Pat
      * @param outputPath  The path to the output file.
      * @param workspacePath The path to the workspace.
      */
-    public static native void buildWorkspace(String outputPath, String workspacePath)
+    private static native void buildWorkspace(String outputPath, String workspacePath, String compression)
             throws com.sensmetry.sysand.exceptions.SysandException;
 
     /**
@@ -184,8 +184,8 @@ public static native void buildWorkspace(String outputPath, String workspacePath
      * @param outputPath  The path to the output file.
      * @param workspacePath The path to the workspace.
      */
-    public static void buildWorkspace(java.nio.file.Path outputPath, java.nio.file.Path workspacePath)
+    public static void buildWorkspace(java.nio.file.Path outputPath, java.nio.file.Path workspacePath, com.sensmetry.sysand.model.CompressionMethod compression)
             throws com.sensmetry.sysand.exceptions.SysandException {
-        buildWorkspace(outputPath.toString(), workspacePath.toString());
+        buildWorkspace(outputPath.toString(), workspacePath.toString(), compression.toString());
     }
 }

bindings/java/java/src/main/java/com/sensmetry/sysand/model/CompressionMethod.java

@@ -0,0 +1,16 @@
+package com.sensmetry.sysand.model;
+
+public enum CompressionMethod {
+    // Store the files as is
+    STORED,
+    // Compress the files using Deflate
+    DEFLATED,
+    /// Compress the files using BZIP2. Only available when sysand is compiled with feature kpar-bzip2
+    BZIP2,
+    /// Compress the files using ZStandard. Only available when sysand is compiled with feature kpar-zstd
+    ZSTD,
+    /// Compress the files using XZ. Only available when sysand is compiled with feature kpar-xz
+    XZ,
+    /// Compress the files using PPMd. Only available when sysand is compiled with feature kpar-ppmd
+    PPMD,
+}

bindings/java/plugin/src/main/java/org/sysand/maven/SysandBuildKParMojo.java

@@ -4,12 +4,16 @@
 
 package org.sysand.maven;
 
+import java.nio.file.Paths;
+
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugins.annotations.LifecyclePhase;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 
+import com.sensmetry.sysand.model.CompressionMethod;
+
 @Mojo(name = "build-kpar", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = false)
 public class SysandBuildKParMojo extends AbstractMojo {
 
@@ -37,6 +41,14 @@ public class SysandBuildKParMojo extends AbstractMojo {
     @Parameter(property = "sysand.outputPath", required = true)
     private String outputPath;
 
+    /**
+     * KPAR compression method. Can be configured as 
+     * {@code <configuration><compressionMethod>...</compressionMethod></configuration>} or
+     * via {@code -Dsysand.compressionMethod=...}.
+     */
+    @Parameter(property = "sysand.compressionMethod", required = false)
+    private String compressionMethod;
+
     @Override
     public void execute() throws MojoExecutionException {
         if (projectPath == null && workspacePath == null) {
@@ -47,14 +59,16 @@ public void execute() throws MojoExecutionException {
             throw new MojoExecutionException("Parameter 'outputPath' must be provided and non-empty");
         }
 
+        CompressionMethod compression = compressionMethod == null ? CompressionMethod.DEFLATED : CompressionMethod.valueOf(compressionMethod.toUpperCase());
+
         try {
             if (workspacePath == null) {
-                getLog().info("Invoking Sysand.buildProject on: " + projectPath + " to " + outputPath);
-                com.sensmetry.sysand.Sysand.buildProject(outputPath, projectPath);
+                getLog().info("Invoking Sysand.buildProject on: " + projectPath + " to " + outputPath + " with compression " + compressionMethod);
+                com.sensmetry.sysand.Sysand.buildProject(Paths.get(outputPath), Paths.get(projectPath), compression);
                 getLog().info("Sysand.buildProject completed successfully.");
             } else {
-                getLog().info("Invoking Sysand.buildWorkspace on: " + workspacePath + " to " + outputPath);
-                com.sensmetry.sysand.Sysand.buildWorkspace(outputPath, workspacePath);
+                getLog().info("Invoking Sysand.buildWorkspace on: " + workspacePath + " to " + outputPath + " with compression " + compressionMethod);
+                com.sensmetry.sysand.Sysand.buildWorkspace(Paths.get(outputPath), Paths.get(workspacePath), compression);
                 getLog().info("Sysand.buildWorkspace completed successfully.");
             }
         } catch (com.sensmetry.sysand.exceptions.SysandException e) {

bindings/java/scripts/java-builder.py

@@ -11,7 +11,7 @@
 import platform
 import shutil
 import subprocess
-from typing import Any
+from typing import Any, Union
 
 
 ROOT_DIR = Path(__file__).absolute().parent.parent.parent.parent
@@ -131,7 +131,7 @@ def compute_full_version(version: str, release_jar_version: bool) -> str:
 
 def build(
     use_release_build: bool,
-    use_existing_native_libs: Path | None,
+    use_existing_native_libs: Union[Path, None],
     sign_artifacts: bool,
     release_jar_version: bool,
     version: str,

bindings/java/src/lib.rs

@@ -11,7 +11,7 @@ use jni::{
 };
 use sysand_core::{
     auth::Unauthenticated,
-    build::KParBuildError,
+    build::{KParBuildError, KparCompressionMethod},
     commands,
     env::local_directory::{self, LocalWriteError},
     info::InfoError,
@@ -321,12 +321,26 @@ fn handle_build_error(env: &mut JNIEnv<'_>, error: KParBuildError<LocalSrcError>
     }
 }
 
+fn compression_from_java_string(
+    env: &mut JNIEnv<'_>,
+    compression: String,
+) -> Option<KparCompressionMethod> {
+    match KparCompressionMethod::try_from(compression) {
+        Ok(compression) => Some(compression),
+        Err(err) => {
+            env.throw_exception(ExceptionKind::SysandException, err.to_string());
+            None
+        }
+    }
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildProject<'local>(
     mut env: JNIEnv<'local>,
     _class: JClass<'local>,
     output_path: JString<'local>,
     project_path: JString<'local>,
+    compression: JString<'local>,
 ) {
     let Some(output_path) = env.get_str(&output_path, "outputPath") else {
         return;
@@ -338,7 +352,14 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildProject<'local>(
         nominal_path: None,
         project_path: Utf8PathBuf::from(project_path),
     };
-    let command_result = sysand_core::commands::build::do_build_kpar(&project, &output_path, true);
+    let Some(compression) = env.get_str(&compression, "compression") else {
+        return;
+    };
+    let Some(compression) = compression_from_java_string(&mut env, compression) else {
+        return;
+    };
+    let command_result =
+        sysand_core::commands::build::do_build_kpar(&project, &output_path, compression, true);
     match command_result {
         Ok(_) => {}
         Err(error) => handle_build_error(&mut env, error),
@@ -351,6 +372,7 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildWorkspace<'local>(
     _class: JClass<'local>,
     output_path: JString<'local>,
     workspace_path: JString<'local>,
+    compression: JString<'local>,
 ) {
     let Some(output_path) = env.get_str(&output_path, "outputPath") else {
         return;
@@ -365,15 +387,26 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_buildWorkspace<'local>(
             return;
         }
     };
+    let Some(compression) = env.get_str(&compression, "compression") else {
+        return;
+    };
+    let Some(compression) = compression_from_java_string(&mut env, compression) else {
+        return;
+    };
     match wrapfs::create_dir_all(&output_path) {
         Ok(_) => {}
         Err(error) => {
             env.throw_exception(ExceptionKind::IOError, error.to_string());
             return;
         }
     }
-    let command_result =
-        sysand_core::commands::build::do_build_workspace_kpars(&workspace, &output_path, true);
+
+    let command_result = sysand_core::commands::build::do_build_workspace_kpars(
+        &workspace,
+        &output_path,
+        compression,
+        true,
+    );
     match command_result {
         Ok(_) => {}
         Err(error) => handle_build_error(&mut env, error),

bindings/js/src/env/local_storage.rs

@@ -178,7 +178,7 @@ impl WriteEnvironment for LocalBrowserStorageEnvironment {
             .read_string(self.versions_path(&uri))
             .map_err(Error::LocalStorage)?;
 
-        let mut kept_versions = "".to_string();
+        let mut kept_versions = String::new();
 
         let mut found = false;
         for current_version in current_versions.lines() {

bindings/py/Cargo.toml

@@ -15,6 +15,10 @@ homepage.workspace = true
 default = ["extension-module", "abi3"]
 extension-module = ["pyo3/extension-module"]
 abi3 = ["pyo3/abi3", "pyo3/abi3-py38"]
+kpar-bzip2 = ["sysand-core/kpar-bzip2", "sysand/kpar-bzip2"]
+kpar-zstd = ["sysand-core/kpar-zstd", "sysand/kpar-zstd"]
+kpar-xz = ["sysand-core/kpar-xz", "sysand/kpar-xz"]
+kpar-ppmd = ["sysand-core/kpar-ppmd", "sysand/kpar-ppmd"]
 
 [dependencies]
 sysand-core = { path = "../../core", features = ["python", "filesystem", "networking"] }

bindings/py/python/sysand/__init__.py

@@ -7,6 +7,7 @@
     InterchangeProjectInfo,
     InterchangeProjectChecksum,
     InterchangeProjectMetadata,
+    CompressionMethod,
 )
 
 from ._info import info_path, info
@@ -47,6 +48,7 @@
     "InterchangeProjectInfo",
     "InterchangeProjectChecksum",
     "InterchangeProjectMetadata",
+    "CompressionMethod",
     ## Add
     "add",
     ## Remove

bindings/py/python/sysand/_build.py

@@ -1,14 +1,22 @@
 from __future__ import annotations
 
+from sysand._model import CompressionMethod
 import sysand._sysand_core as sysand_rs  # type: ignore
 
 from pathlib import Path
 
 
-def build(output_path: str | Path, project_path: str | Path | None = None) -> None:
+def build(
+    output_path: str | Path,
+    project_path: str | Path | None = None,
+    compression: CompressionMethod | None = None,
+) -> None:
     if project_path is not None:
         project_path = str(project_path)
-    sysand_rs.do_build_py(str(output_path), project_path)
+
+    # comp = None if compression is None else _convert_compression(compression)
+    comp = None if compression is None else compression.name
+    sysand_rs.do_build_py(str(output_path), project_path, comp)
 
 
 __all__ = [