Added HDF5 compression to EmpCylSL disk cache files

.gitignore

@@ -1,6 +1,11 @@
 
 build/*
 
+# Python cache files
+__pycache__/
+*.py[cod]
+*$py.class
+
 Makefile
 *.lo
 *.a

_codeql_detected_source_root

@@ -0,0 +1 @@
+.

expui/BiorthBasis.H

@@ -948,8 +948,12 @@ namespace BasisClasses
   };
 
   /**
-     Uses EmpCylSL basis to evaluate expansion coeffients and provide
+     @brief Uses EmpCylSL basis to evaluate expansion coeffients and provide
      potential and density basis fields
+
+     @note The HDF5 disk basis cache is gzip compressed by default to
+     save disk space. The user may turn off compression using the
+     `compress: 0` parameter, although we have no particular use case.
   */
   class Cylindrical : public BiorthBasis
   {

expui/BiorthBasis.cc

@@ -1261,7 +1261,9 @@ namespace BasisClasses
     "playback",
     "coefCompute",
     "coefMaster",
-    "pyname"
+    "pyname",
+    "compress",
+    "shuffle"
   };
 
   Cylindrical::Cylindrical(const YAML::Node& CONF) :
@@ -1591,6 +1593,15 @@ namespace BasisClasses
       sl->set_covar(true);
     }
 
+    // Set HDF5 compression parameters for cache files
+    //
+    if (conf["compress"]) {
+      unsigned compress = conf["compress"].as<unsigned>();
+      bool shuffle = true;
+      if (conf["shuffle"]) shuffle = conf["shuffle"].as<bool>();
+      sl->setH5Params(compress, shuffle);
+    }
+
     // Cache override for old Eigen cache
     //
     if (oldcache) sl->AllowOldCache();
@@ -5155,10 +5166,11 @@ namespace BasisClasses
     auto dcpl3 = HighFive::DataSetCreateProps{}; // covariance
 
     // Properties for sample stats
-    if (H5compress) {
+    // Note: Sample stats only use Deflate compression, not Szip
+    if (H5compress > 0) {
       unsigned int csz = sampleCounts.size();
       dcpl1.add(HighFive::Chunking({csz, 1}));
-      if (H5shuffle) dcpl1.add(HighFive::Shuffle());
+      if (H5shuffle && H5compress > 0) dcpl1.add(HighFive::Shuffle());
       dcpl1.add(HighFive::Deflate(H5compress));
     }
 
@@ -5196,7 +5208,7 @@ namespace BasisClasses
     stanza.createAttribute<unsigned>
       ("fullCovar", HighFive::DataSpace::From(icov)).write(icov);
 
-    if (H5compress) {
+    if (H5compress > 0 || H5szip) {
       // Szip parameters
       const int options_mask = H5_SZIP_NN_OPTION_MASK;
       const int pixels_per_block = 8;
@@ -5207,7 +5219,7 @@ namespace BasisClasses
       HighFive::Chunking data_dims2{std::min<unsigned>(csz2, H5chunk), 1};
 
       dcpl2.add(data_dims2);
-      if (H5shuffle) dcpl2.add(HighFive::Shuffle());
+      if (H5shuffle && (H5compress > 0 || H5szip)) dcpl2.add(HighFive::Shuffle());
       if (H5szip) {
 	dcpl2.add(HighFive::Szip(options_mask, pixels_per_block));
       } else {
@@ -5220,7 +5232,7 @@ namespace BasisClasses
       HighFive::Chunking data_dims3{std::min<unsigned>(csz3, H5chunk), 1};
 
       dcpl3.add(data_dims3);
-      if (H5shuffle) dcpl3.add(HighFive::Shuffle());
+      if (H5shuffle && (H5compress > 0 || H5szip)) dcpl3.add(HighFive::Shuffle());
       if (H5szip) {
 	dcpl3.add(HighFive::Szip(options_mask, pixels_per_block));
       } else {

exputil/EmpCylSL.cc

@@ -7377,8 +7377,26 @@ void EmpCylSL::WriteH5Cache()
     file.createAttribute<double>     ("hscl",    HighFive::DataSpace::From(HSCALE)).  write(HSCALE);
     file.createAttribute<double>     ("cmass",   HighFive::DataSpace::From(cylmass)). write(cylmass);
 
-    // Cosine functions
+    // Enable compression
+    //
+    auto dcpl = HighFive::DataSetCreateProps{};
+
+    if (H5compress > 0 || H5szip) {
+      // Chunking
+      unsigned long nx = NUMX + 1, ny = NUMY + 1;
+      dcpl.add(HighFive::Chunking({nx, ny}));
+      if (H5shuffle && (H5compress > 0 || H5szip)) dcpl.add(HighFive::Shuffle());
+      if (H5szip) {
+        const int options_mask = H5_SZIP_NN_OPTION_MASK;
+        const int pixels_per_block = 8;
+        dcpl.add(HighFive::Szip(options_mask, pixels_per_block));
+      } else {
+        dcpl.add(HighFive::Deflate(H5compress));
+      }
+    }
 
+    // Cosine functions
+    //
     auto cosine = file.createGroup("Cosine");
 
     // Harmonic order
@@ -7394,15 +7412,15 @@ void EmpCylSL::WriteH5Cache()
 	sout << n;
 	auto order = harmonic.createGroup(sout.str());
 
-	order.createDataSet("potC",    potC   [m][n]);
-	order.createDataSet("rforceC", rforceC[m][n]);
-	order.createDataSet("zforceC", zforceC[m][n]);
-	order.createDataSet("densC",   densC  [m][n]);
+	order.createDataSet("potC",    potC   [m][n], dcpl);
+	order.createDataSet("rforceC", rforceC[m][n], dcpl);
+	order.createDataSet("zforceC", zforceC[m][n], dcpl);
+	order.createDataSet("densC",   densC  [m][n], dcpl);
       }
     }
 
     // Sine functions
-
+    //
     auto sine = file.createGroup("Sine");
 
     // Harmonic order
@@ -7418,10 +7436,10 @@ void EmpCylSL::WriteH5Cache()
 	sout << n;
 	auto order = harmonic.createGroup(sout.str());
 
-	order.createDataSet("potS",    potS   [m][n]);
-	order.createDataSet("rforceS", rforceS[m][n]);
-	order.createDataSet("zforceS", zforceS[m][n]);
-	order.createDataSet("densS",   densS  [m][n]);
+	order.createDataSet("potS",    potS   [m][n], dcpl);
+	order.createDataSet("rforceS", rforceS[m][n], dcpl);
+	order.createDataSet("zforceS", zforceS[m][n], dcpl);
+	order.createDataSet("densS",   densS  [m][n], dcpl);
       }
     }
 

include/EmpCylSL.H

@@ -241,6 +241,17 @@ protected:
   //! Cache versioning
   inline static const std::string Version = "1.0";
 
+  //@{
+  //! H5 compression
+  /*! 
+   * H5compress: Compression level for HDF5 cache.
+   * Valid range: 0-9 (0 = no compression, 9 = maximum compression)
+   */
+  unsigned H5compress = 5;
+  bool     H5shuffle  = true;
+  bool     H5szip     = false;
+  //@}
+
   //! The cache file name
   std::string cachefile;
 				// 1=write, 0=read
@@ -934,6 +945,36 @@ public:
     }
   }
 
+  //! Set HDF5 compression parameters for cache files
+  //! @param compress Compression level for deflate/gzip (valid range: 0-9, where 0=no compression, 9=max compression)
+  //! @param shuffle Enable byte shuffle filter (default: true)
+  //! @param szip Enable szip compression (default: false)
+  //! @note Szip and deflate compression are mutually exclusive. When both are configured, deflate compression takes precedence and the szip parameter is ignored.
+  void setH5Params(unsigned compress, bool shuffle=true, bool szip=false)
+  {
+    // Validate compression level (unsigned type prevents negative values)
+    if (compress > 9) {
+      if (myid==0) {
+	std::cout << "EmpCylSL: compression level " << compress
+		  << " out of range [0,9], using 9" << std::endl;
+      }
+      compress = 9;
+    }
+
+    // Warn if both szip and deflate compression are configured
+    if (szip && compress > 0) {
+      if (myid==0) {
+	std::cout << "EmpCylSL: szip and deflate compression are mutually exclusive. "
+		  << "Using deflate; szip parameter will be ignored." << std::endl;
+      }
+      szip = false;  // Disable szip when both are configured
+    }
+
+    H5compress = compress;
+    H5shuffle  = shuffle;
+    H5szip     = szip;
+  }
+
   vector<double> sanity() { 
     vector<double> ret;
     for (int m=0; m<=MMAX; m++) ret.push_back(accum_cos[0][m]);

pyEXP/BasisWrappers.cc

@@ -87,7 +87,10 @@ void BasisFactoryClasses(py::module &m)
     or with the same basis used on previous pyEXP invocations.  At this
     point, you must create the YAML configuration for the basis even if
     the basis is cached.  This is a safety and consistency feature that
-    may be relaxed in a future version.
+    may be relaxed in a future version.  The HDF5 disk basis cache is
+    gzip compressed by default to save disk space. The user may turn off
+    compression using the `compress: 0` parameter, although we have no
+    particular use case.
 
     Coefficient creation
     --------------------
@@ -1504,7 +1507,7 @@ void BasisFactoryClasses(py::module &m)
          Returns
          -------
          None
-         )", py::arg("compress")=5, py::arg("chunkSize")=1024*1024, py::arg("shuffle")=true, py::arg("azip")=false)
+         )", py::arg("compress")=5, py::arg("chunkSize")=1024*1024, py::arg("shuffle")=true, py::arg("szip")=false)
     .def("makeFromFunction", &BasisClasses::BiorthBasis::makeFromFunction,
 	 py::call_guard<py::gil_scoped_release>(),
 	 R"(

src/Component.cc

@@ -2510,21 +2510,21 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs)
     }
 
     dcpl1.add(HighFive::Chunking(chunk));
-    if (H5shuffle) dcpl1.add(HighFive::Shuffle());
+    if (H5shuffle && H5compress > 0) dcpl1.add(HighFive::Shuffle());
     dcpl1.add(HighFive::Deflate(H5compress));
 
     dcpl3.add(HighFive::Chunking(chunk, 3));
-    if (H5shuffle) dcpl3.add(HighFive::Shuffle());
+    if (H5shuffle && H5compress > 0) dcpl3.add(HighFive::Shuffle());
     dcpl3.add(HighFive::Deflate(H5compress));
 
     if (niattrib) {
       dcplI.add(HighFive::Chunking(chunk, niattrib));
-      if (H5shuffle) dcplI.add(HighFive::Shuffle());
+      if (H5shuffle && H5compress > 0) dcplI.add(HighFive::Shuffle());
       dcplI.add(HighFive::Deflate(H5compress));
     }
     if (ndattrib) {
       dcplD.add(HighFive::Chunking(chunk, ndattrib));
-      if (H5shuffle) dcplD.add(HighFive::Shuffle());
+      if (H5shuffle && H5compress > 0) dcplD.add(HighFive::Shuffle());
       dcplD.add(HighFive::Deflate(H5compress));
     }
   }
@@ -2652,7 +2652,7 @@ void Component::write_H5(H5::Group& group)
       if (H5compress) dcpl.setDeflate(H5compress);
 
       // Enable shuffle filter
-      if (H5shuffle) dcpl.setShuffle();
+      if (H5shuffle && H5compress > 0) dcpl.setShuffle();
     }
 
     // Create dataspace

src/Cylinder.H

@@ -122,6 +122,11 @@ class MixtureBasis;
 
     @param dumpbasis set to true dumps the basis to a file for
     diagnostic examination (default: false)
+
+    @param compress sets the HDF5 cache file compression level.  The
+     HDF5 disk basis cache is gzip compressed by default at compress=5
+     to save disk space. The user may turn off compression using
+     compress=0, although we have no particular use case.
 */
 class Cylinder : public Basis
 {

src/Cylinder.cc

@@ -74,7 +74,9 @@ Cylinder::valid_keys = {
   "coefCompute",
   "coefMaster",
   "pyname",
-  "dumpbasis"
+  "dumpbasis",
+  "compress",
+  "shuffle"
 };
 
 Cylinder::Cylinder(Component* c0, const YAML::Node& conf, MixtureBasis *m) :
@@ -184,6 +186,14 @@ Cylinder::Cylinder(Component* c0, const YAML::Node& conf, MixtureBasis *m) :
   if (EVEN_M)   ortho->setEven(EVEN_M);
   ortho->setSampT(defSampT);
 
+  // Set HDF5 compression parameters for cache files
+  if (conf["compress"]) {
+    unsigned compress = conf["compress"].as<unsigned>();
+    bool shuffle = true;
+    if (conf["shuffle"]) shuffle = conf["shuffle"].as<bool>();
+    ortho->setH5Params(compress, shuffle);
+  }
+
   try {
     if (conf["tk_type"]) ortho->setTK(conf["tk_type"].as<std::string>());
   }

tests/CMakeLists.txt

@@ -26,6 +26,13 @@ if(ENABLE_PYEXP)
     ${PYTHON_EXECUTABLE} cyl_basis.py
     WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/Disk")
 
+  # Test HDF5 compression functionality in cylindrical basis
+  add_test(NAME pyexpCylCompressionTest
+    COMMAND ${CMAKE_COMMAND} -E env
+    PYTHONPATH=${CMAKE_BINARY_DIR}/pyEXP:$ENV{PYTHONPATH}
+    ${PYTHON_EXECUTABLE} cyl_compression_test.py
+    WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/Disk")
+
   # Remove cache files
   add_test(NAME removeSphCache
     COMMAND ${CMAKE_COMMAND} -E remove .slgrid_sph_cache
@@ -45,7 +52,7 @@ if(ENABLE_PYEXP)
 
   # Set labels for pyEXP tests
   set_tests_properties(pyexpLoadTest pyexpSphBasisTest PROPERTIES LABELS "quick")
-  set_tests_properties(pyexpCylBasisTest PROPERTIES LABELS "long")
+  set_tests_properties(pyexpCylBasisTest pyexpCylCompressionTest PROPERTIES LABELS "long")
 
 endif()