MacOS fixes + troubleshooting documentation

.github/workflows/Check-Cyz2Json.yml

@@ -5,10 +5,12 @@ on:
     branches: [ "main" ]
     paths-ignore:
       - '.github/**'
+      - 'README.md'
   pull_request:
     branches: [ "main" ]
     paths-ignore:
       - '.github/**'
+      - 'README.md'
   workflow_dispatch:
   schedule:
     - cron: '00 2 16 * *'
@@ -27,13 +29,24 @@ jobs:
       uses: actions/setup-dotnet@v5
       with:
         dotnet-version: 8.0.x
-
-    - name: Restore dependencies and Build
+
+    - name: Install OpenCvSharp dependencies for Linux
+      if: matrix.os == 'ubuntu-latest'
       shell: bash
       run: |
-        dotnet restore
-        dotnet build --no-restore -p:OutputPath=./bin/
-
+        # Dependencies needed on Linux
+        # They need to be installed here to avoid compiling the project with outdated versions which won't be found locally
+        sudo apt-get update
+        sudo apt-get install -y \
+          libgtk-3-dev \
+          libtesseract-dev \
+          libavcodec-dev \
+          libavformat-dev \
+          libavutil-dev \
+          libswscale-dev \
+          libopenexr-dev \
+          libtiff-dev
+
     - name: Set RID
       id: set-rid
       shell: bash
@@ -49,15 +62,41 @@ jobs:
             fi
             ;;
         esac
+
+    - name: Restore dependencies and Build
+      shell: bash
+      run: |
+        dotnet restore
+        dotnet build --no-restore -p:OutputPath=./bin/
 
     - name: Publish
       shell: bash
       run: |
         dotnet publish \
           -c Release \
           -r ${{ steps.set-rid.outputs.rid }} \
-          --sc true \
-          -p:IncludeNativeLibrariesForSelfExtract=true \
-          -p:PublishReadyToRun=true \
-          -p:OutputPath=./bin/${{ steps.set-rid.outputs.rid }}/publish/
-        echo "Build succeeded !"
+          -p:OutputPath=./bin/${{ steps.set-rid.outputs.rid }}/
+
+    - name: Create symlink for OpenCvSharpExtern.dylib
+      if: matrix.os == 'macos-latest'
+      shell: bash
+      run: |
+        publish_dir="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish"
+        cd $publish_dir/
+        # Create symlink to redirect calls to OpenCvSharpExtern.dylib to libOpenCvSharpExtern.dylib (the file actually created)
+        ln -sf libOpenCvSharpExtern.dylib OpenCvSharpExtern.dylib
+        cd ../../..
+
+    - name: Run executable
+      shell: bash
+      run: |
+        exmpl_path="examples/data/"
+        file="$exmpl_path/SeineMO_Microphoto_station02_flr26_2ul_8min 2024-05-04 07h54_SubSet.cyz"
+        dll_path="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish/Cyz2Json.dll"
+        bn=$(basename "$file")
+        # if macOS, use env var to look for native dependencies inside publish dir (or else it will throw an error)
+        if [ "${{ matrix.os }}" == "macos-latest" ]; then
+          export DYLD_FALLBACK_LIBRARY_PATH=$(dirname $dll_path)
+        fi
+        dotnet $dll_path "$file" --imaging-set-information --image-processing --output $exmpl_path/"$bn".json
+        echo "File successfully converted !"

.github/workflows/Release-Cyz2Json.yml

@@ -94,58 +94,24 @@ jobs:
         publish_dir="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish"
         cp LICENSE.txt "$publish_dir/"
 
-    - name: Prepare asset directory
-      shell: bash
-      run: |
-        cd Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/
-        mkdir -p asset
-        mv publish asset/
-
     - name: Create symlink for OpenCvSharpExtern.dylib
       if: matrix.os == 'macos-latest'
       shell: bash
       run: |
-        publish_dir="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/asset/publish"
+        publish_dir="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish"
         cd $publish_dir/
         # Create symlink to redirect calls to OpenCvSharpExtern.dylib to libOpenCvSharpExtern.dylib (the file actually created)
         ln -sf libOpenCvSharpExtern.dylib OpenCvSharpExtern.dylib
 
-    - name: Fix library path (macOS)
-      if: matrix.os == 'macos-latest'
-      shell: bash
-      run: |
-        publish_dir="Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/asset/publish"
-        cd $publish_dir/
-        # Native libs are searched in @executable_path/../libs/ by default but this folder doesn't exist
-        # Get dependencies to fix (starting with @executable_path/../libs)
-        deps=$(
-          otool -L OpenCvSharpExtern.dylib |
-          grep "@executable_path/../libs/" |
-          sed -E 's/\(.*\)//' |
-          awk -F'/' '{print $NF}' |
-          sort |
-          uniq
-        )
-        # Move those libs to the newly created libs folder
-        mkdir -p ../libs
-        for dep in $deps; do
-          if [ -n "$dep" ]; then
-            echo "Moving $dep to ../libs/"
-            mv "$dep" "../libs/"
-          fi
-        done
-        echo "Libs path"
-        otool -L OpenCvSharpExtern.dylib
-
     - name: Create Release Archive
       shell: pwsh
       run: |
         if ($env:RUNNER_OS -eq "Windows") {
-          cd ./Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/asset
+          cd ./Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish
           7z a ../../../../cyz2json-${{ matrix.os }}.zip ./*
         }
         else {
-          cd ./Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/asset
+          cd ./Cyz2Json/bin/${{ steps.set-rid.outputs.rid }}/publish
           zip -r ../../../../cyz2json-${{ matrix.os }}.zip ./*
         }
 

README.md

@@ -6,12 +6,9 @@
 
 ## Introduction
 
-This program converts flow cytometry data stored in a CytoBuoy CYZ
-format file to JSON format.
+This program converts flow cytometry data stored in a CytoBuoy CYZ format file to JSON format.
 
-Although CYZ files work with the Cytobuoy supplied CytoClus program,
-we want to access the data from a wider set of data science tooling
-including Python and R. We therefore convert to JSON.
+Although CYZ files work with the Cytobuoy supplied CytoClus program, we want to access the data from a wider set of data science tooling including Python and R. We therefore convert to JSON.
 
 This program uses the [CytoBuoy CyzFile API](https://github.com/Cytobuoy/CyzFile-API) and targets the Microsoft
 .NET 8 framework on Windows, macOS or Linux.
@@ -22,8 +19,7 @@ This program uses the [CytoBuoy CyzFile API](https://github.com/Cytobuoy/CyzFile
 
 #### Installation
 The first step is to install the Microsoft dotnet runtime version 8.
-Microsoft provides detailed instructions at
-https://learn.microsoft.com/en-us/dotnet/core/install/.
+Microsoft provides detailed instructions at https://learn.microsoft.com/en-us/dotnet/core/install/.
 
 For example, installation on Linux Ubuntu 22.04 is as follows:
 
@@ -41,8 +37,7 @@ To generate the version number, you will need to install the `nbgv` tool.
 You can install this tool by typing:
 `dotnet tool install -g nbgv`
 
-The `Cyz2Json` program can then be compiled by cloning the project
-from GitHub and typing `dotnet build -p:OutputPath=./bin/`.
+The `Cyz2Json` program can then be compiled by cloning the project from GitHub and typing `dotnet build -p:OutputPath=./bin/`.
 The default version is the one given by the [Cyz2Json.csproj](https://github.com/OBAMANEXT/cyz2json/blob/main/Cyz2Json/Cyz2Json.csproj/) file (unless passed directly during build via -p:Version), the assembly informal version is  "Cyz2Json-the current Version-the latest Git SHA".
 
 ### Build with GitHub actions
@@ -121,8 +116,7 @@ Cyz2Json.exe input.cyz --output output.json
 
 ### Bulk file conversion
 
-We often find ourselves needing to undertake a bulk conversion of a
-large set of files. The following shell script can help:
+We often find ourselves needing to undertake a bulk conversion of a large set of files. The following shell script can help:
 
 ```
 #!/usr/bin/env bash
@@ -132,6 +126,64 @@ do
 done
 ```
 
+## Troubleshooting 
+### On OpenCvSharpExtern
+
+You may run into the `System.DllNotFoundException: Unable to load shared library 'OpenCvSharpExtern' or one of its dependencies.` when trying to run the program. In that case, either the library OpenCvSharpExtern and/or its dependencies are missing or the path to load them is wrong.  Here are a few steps to help you deal with this problem if you ever run into it.
+
+#### Linux
+
+The `libOpenCvSharpExtern.so` library should be exported by the `OpenCvSharp4.official.runtime.linux-x64` nugget package and found either in the latest release asset or in your local build results. It is statically linked to its dependencies which are not themselves exported. Most of those libraries are pre-installed in standard ubuntu distributions, but can be missing in minimal environment/containers. Besides, if you are using the latest release asset, the version of the dependencies you have locally might not be compatible with the versions linked to the library exported in the asset (should be latest "dev" versions).
+
+To fix this, the easiest way is to :
+- move to your binaries folder (either the downloaded asset or the result of the local build) and run the `ldd libOpenCvSharpExtern.so` command, this will list all statically linked dependencies and highlight missing ones as "not found",
+- then update and/or install those libraries.
+
+Example of installation :
+```
+sudo apt-get update
+sudo apt-get install -y \
+  libgtk-3-dev \
+  libtesseract-dev \
+  libavcodec-dev \
+  libavformat-dev \
+  libavutil-dev \
+  libswscale-dev \
+  libopenexr-dev \
+  libtiff-dev
+```
+
+#### macOS
+
+The `libOpenCvSharpExtern.dylib` library should be exported with its dependencies either by the `OpenCvSharp4.runtime.osx.10.15-x64` or the `OpenCvSharp4.runtime.osx_arm64` nugget packages. A [known issue](https://github.com/shimat/opencvsharp/issues/1797) with the macOS (arm64) version of the library is its hard dependency on the `@executable_path/../dirs` path, basically, it looks for native dependencies in this folder, that doesn't exist, instead of the folder where they are actually exported (the same as the `libOpenCvSharpExtern.dylib` file). You can check that by running the command `otool -L libOpenCvSharpExtern.dylib` and see the list of all dependencies and where the program will search them.
+
+The easiest way to deal with this issue is by using an environment variable (via a shell script) :
+```
+#!/usr/bin/env bash
+dll_path="bin/Cyz2Json.dll"
+export DYLD_FALLBACK_LIBRARY_PATH=$(dirname $dll_path)
+dotnet $dll_path input.cyz --output output.json
+```
+
+This will tell the program to try the path given by `DYLD_FALLBACK_LIBRARY_PATH` if all other (wrong) paths fail. In the example above, the native libraries are located in the same folder as the Cyz2Json.dll (dll_path), if it is not the case for you (for example they are in `runtimes/osx-arm64/native/` folder) then simply modify the script to match your local directory structure.
+
+Another issue is that the program is looking for an `OpenCvSharpExtern.dylib` file instead of the correct `libOpenCvSharpExtern.dylib`. The easiest way to correct this error is by creating a symlink : `ln -sf libOpenCvSharpExtern.dylib OpenCvSharpExtern.dylib` so any call to `OpenCvSharpExtern.dylib` will redirect to `libOpenCvSharpExtern.dylib`. The symlink was already created for the latest release asset and should be preserved.
+
+### On Garbage Collector
+
+You may run into the following error on Linux when trying to execute the program : 
+```
+GC heap initialization failed with error 0x8007000E
+Failed to create CoreCLR, HRESULT: 0x8007000E`
+```
+
+This problem is machine specific, check with the command `ulimit -a` that your `virtual memory` and `max memory size` are set either to `unlimited` or a large enough number. You can also check `cat /proc/sys/vm/overcommit_memory`, the value should be 0 (overcommit reasonable memory request on the heuristics base, default) or 1 (always overcommit). If it is set to 2 then it means never overcommit.
+
+To bypass those problems, you can :
+- temporarily set the virtual memory to unlimited `ulimit -v unlimited`,
+- or temporarily change the overcommit mode to 1 (or 0) `echo 1 | sudo tee /proc/sys/vm/overcommit_memory`,
+- you can also set an environment variable to temporarily lower the memory usage of the program (but be mindful of the value used if it is too small) : `export DOTNET_GCHeapHardLimit=1C0000000`.
+
 ## Processing CYZ JSON files 
 
 ### Using Python
@@ -145,9 +197,7 @@ data = json.load(open("pond.json", encoding="utf-8-sig"))
 print(data)
 ```
 
-Note the need to explicitly specify the encoding to deal with
-Microsoft's and Python's differences in interpretation of the
-standards regarding byte order marks in UTF-8 files.
+Note the need to explicitly specify the encoding to deal with Microsoft's and Python's differences in interpretation of the standards regarding byte order marks in UTF-8 files.
 
 ### R example
 
@@ -164,25 +214,19 @@ print(json_data)
 
 ### On images
 
-If a CYZ file contains images, we currently base64 encode them and
-include them inline in the JSON. This is costly in terms of disk and
-only time will tell if it is a sensible design decision. A future
-enhancement would be to include a flag that writes the files out as
-JPEG images.
+If a CYZ file contains images, we currently base64 encode them and include them inline in the JSON. This is costly in terms of disk and only time will tell if it is a sensible design decision. A future enhancement would be to include a flag that writes the files out as JPEG images.
 
 Note that the images are un-cropped by default. The `--image-processing` option can now be used to process and crop images when exporting. 
-Initially, the tool didn't support cropping as an expedience to allow
-cross platform operation (The CyzFile-API only supports cropping on
-Windows platforms).
+Initially, the tool didn't support cropping as an expedience to allow cross platform operation (The CyzFile-API only supports cropping on Windows platforms).
 
 ### On the API
 
-Cyz2json is using a dll from the [CyzFile-API](https://github.com/Cytobuoy/CyzFile-API), at this moment, using version 1.6.1.0.
+Cyz2json is using a dll from the [CyzFile-API](https://github.com/Cytobuoy/CyzFile-API).
 
 To update you can use the [API-update-from-tags.yml](https://github.com/OBAMANEXT/cyz2json/blob/main/.github/workflows/API-update-from-tags.yml/) workflow which fetches the latest version of the API (in tags), you can run it manually or let it work on schedule (15th of each month), in that case, it compares the latest tag with the current version in this repo to choose whether to update or not.
 
 You can also update manually without the workflow, to do so :
-- download the latest net8.0 release from https://github.com/Cytobuoy/CyzFile-API/releases,
+- download the latest release from https://github.com/Cytobuoy/CyzFile-API/releases,
 - unzip the downloaded file,
 - copy the files from the dll folder to the CyzFile folder,
 - rebuild the project.
@@ -199,21 +243,12 @@ My thanks to the following organisations for supporting this work:
 
 [CytoBuoy b.v.](https://www.cytobuoy.com/)
 
-I am grateful to Rob Lievaart at Cytobuoy for providing the libraries,
-code and examples on which this software is based. The CyzFile-API is
-licensed under the terms described in CyzFile-API_LICENSE.TXT.
+I am grateful to Rob Lievaart at Cytobuoy for providing the libraries, code and examples on which this software is based. The CyzFile-API is licensed under the terms described in CyzFile-API_LICENSE.TXT.
 
 Thanks to Eric Payne at Cefas for Visual Studio wizardry.
 
 ## Disclaimers
 
-The [OBAMA-NEXT](https://obama-next.eu/) project has been approved under
-HORIZON-CL6-2022-BIODIV-01-01: Observing and mapping biodiversity and
-ecosystems, with particular focus on coastal and marine ecosystems
-(Grant Agreement 101081642). Funded by the European Union and UK
-Research and Innovation. Views and opinions expressed are however
-those of the authors only and do not necessarily reflect those of the
-European Union or UK Research and Innovation. Neither the European
-Union nor the granting authority can be held responsible for them.
+The [OBAMA-NEXT](https://obama-next.eu/) project has been approved under HORIZON-CL6-2022-BIODIV-01-01: Observing and mapping biodiversity and ecosystems, with particular focus on coastal and marine ecosystems (Grant Agreement 101081642). Funded by the European Union and UK Research and Innovation. Views and opinions expressed are however those of the authors only and do not necessarily reflect those of the European Union or UK Research and Innovation. Neither the European Union nor the granting authority can be held responsible for them.