Feature upgrades

README.adoc

@@ -74,6 +74,13 @@ WARNING: WebUI conversion is experimental and may be slower or less stable than
 
 WARNING: The API is still evolving. Endpoints, parameters, and response formats may change.
 
+Interactive OpenAPI docs are available at:
+
+* `http://localhost:5000/api/v1/`
+
+For the Python client-oriented endpoint contract used in `jvm-api-v1`, see:
+`docs/jvm_api_v1.md`.
+
 Example (Python) for fetching a submatrix tile as an image:
 
 ```python
@@ -97,12 +104,19 @@ png_data_url = data["image"]
 print(png_data_url[:64])
 ```
 
-To apply visualization/normalization settings before fetching tiles:
+To apply visualization settings before fetching tiles:
 
 * POST `/set_visualization_options` with visualization parameters.
-* POST `/set_normalization` with normalization settings.
+* Optionally POST `/render_pipeline/set` for custom graph pipeline.
 * Then call `/get_tile` as shown above.
 
+For tensor workflows (NumPy/Torch), numeric submatrices are available via:
+
+* `POST /matrix/query`
+** units: `PIXELS`, `BINS`, `BP`
+** signal modes: `RAW_COUNTS`, `COOLER_WEIGHTED`, `TRADITIONAL_NORMALIZED`, `PIPELINE_SIGNAL`
+** binary formats for fast transfer: `BINARY_FLOAT32`, `BINARY_FLOAT64`, `BINARY_INT64`
+
 === Supported platforms / JDK details
 
 * *OS/CPU (prebuilt libs):* Linux (glibc) and Windows, AMD64.

build.gradle.kts

@@ -71,7 +71,7 @@ val webUIRepositoryDirectory =
   if (localWebUIRepositoryDirectory.asFile.exists()) localWebUIRepositoryDirectory else remoteWebUIRepositoryDirectory
 val webUIRepositoryAddress = "https://github.com/ctlab/HiCT_WebUI.git"
 val webUITargetDirectory = layout.projectDirectory.dir("src/main/resources/webui")
-val webUIBranch = "migrate-converters-update-ui-1dtracks"
+val webUIBranch = "master"
 
 version = readVersion()
 

docs/jvm_api_v1.md

@@ -0,0 +1,114 @@
+# HiCT_JVM API Reference for Python client (`hict_jvm_api`)
+
+This document describes the endpoint subset used by the `jvm-api-v1` Python library.
+All endpoints are served by HiCT_JVM API server (`start-api-server` or `start-server`).
+
+Interactive documentation is served by the backend itself:
+
+- `GET /api/v1/` - Swagger UI
+- `GET /api/v1/openapi.yaml` - OpenAPI v1 source
+
+## Base URL
+
+Typically:
+
+- `http://localhost:5000`
+- `http://localhost:5001`
+
+## Core endpoints
+
+### Session and files
+
+| Method | Path | Purpose |
+|---|---|---|
+| `GET` | `/version` | Server/version metadata |
+| `POST` | `/list_files` | List files under `DATA_DIR` |
+| `POST` | `/list_files_detailed` | List files with metadata |
+| `POST` | `/list_coolers` | List `.cool/.mcool` files |
+| `POST` | `/list_fasta_files` | List FASTA files |
+| `POST` | `/list_agp_files` | List AGP files |
+| `POST` | `/open` | Open primary HiCT source |
+| `POST` | `/open_progress` | Read open progress |
+| `POST` | `/attach` | Attach existing in-memory session |
+| `POST` | `/close` | Close active session |
+
+### Secondary source
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/secondary/status` | Read secondary-source status |
+| `POST` | `/secondary/open` | Attach secondary source (`allowMismatch` supported) |
+| `POST` | `/secondary/close` | Detach secondary source |
+| `POST` | `/secondary/set_assembly_source` | Set `PRIMARY` or `SECONDARY` assembly source |
+
+### Tiles and rendering
+
+| Method | Path | Purpose |
+|---|---|---|
+| `GET` | `/get_tile` | Fetch rendered tile/region |
+| `POST` | `/matrix/query` | Fetch numeric submatrix (raw/weighted/normalized/pipeline signal) |
+| `POST` | `/tiles/reload` | Invalidate tile caches and bump versions |
+| `POST` | `/get_visualization_options` | Get current visualization options |
+| `POST` | `/set_visualization_options` | Set visualization options |
+| `POST` | `/render_pipeline/get` | Get custom rendering pipeline graph |
+| `POST` | `/render_pipeline/set` | Set custom rendering pipeline graph |
+| `POST` | `/render_pipeline/reset` | Reset pipeline to defaults |
+
+`/get_tile` supported formats:
+
+- `JSON_PNG_WITH_RANGES` (JSON with base64 PNG + ranges)
+- `PNG` (raw PNG bytes)
+- `PNG_BY_PIXELS` (raw PNG bytes for arbitrary pixel-space region)
+
+`/matrix/query` supports:
+
+- Units: `PIXELS`, `BINS`, `BP`
+- Signal modes: `RAW_COUNTS`, `COOLER_WEIGHTED`, `TRADITIONAL_NORMALIZED`, `PIPELINE_SIGNAL`
+- Response formats: `BINARY_FLOAT32`, `BINARY_FLOAT64`, `BINARY_INT64`, `JSON`
+
+### Scaffolding operations
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/reverse_selection_range` | Reverse assembly interval |
+| `POST` | `/move_selection_range` | Move interval |
+| `POST` | `/split_contig_at_bin` | Split contig at pixel/bin coordinate |
+| `POST` | `/group_contigs_into_scaffold` | Group interval into scaffold |
+| `POST` | `/ungroup_contigs_from_scaffold` | Ungroup scaffold interval |
+| `POST` | `/move_selection_to_debris` | Move interval to debris |
+
+### FASTA / AGP
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/link_fasta` | Link FASTA to current source |
+| `POST` | `/get_fasta_for_assembly` | Export FASTA for full assembly |
+| `POST` | `/get_fasta_for_selection` | Export FASTA for selected rectangle |
+| `POST` | `/get_agp_for_assembly` | Export AGP |
+| `POST` | `/load_agp` | Import AGP and update assembly state |
+
+### Conversion jobs
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/convert/jobs` | Submit single conversion job |
+| `POST` | `/convert/jobs/batch` | Submit batch conversion jobs |
+| `POST` | `/convert/jobs/list` | List all conversion jobs |
+| `GET` | `/convert/jobs/:jobId` | Get single job status |
+| `POST` | `/convert/jobs/:jobId/stop` | Cancel running job |
+| `GET` | `/convert/download/:jobId` | Download completed conversion output |
+
+`/convert/jobs` and `/convert/jobs/batch` support `overwrite` boolean.
+
+### Diagnostics
+
+| Method | Path | Purpose |
+|---|---|---|
+| `POST` | `/diagnostics/workers` | Worker-pool and queue diagnostics |
+
+## Notes for Python client authors
+
+- Session state is server-side and mutable; reopen/attach as needed.
+- Scaffolding operations update assembly state and bump tile generations.
+- `PIXELS` are visible pixels (hidden contigs excluded) at selected resolution.
+- For throughput-sensitive region fetches use `GET /get_tile?format=PNG_BY_PIXELS` with persistent HTTP client reuse.

ldbg.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 export VERTXWEB_ENVIRONMENT="dev"
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-VXPORT=5000 DATA_DIR="/mnt/Models/HiCT/data/" TILE_SIZE=256 java -jar ${SCRIPT_DIR}/build/libs/hict_server-*-fat.jar $@
+VXPORT=5000 DATA_DIR="/mnt/Models/HiCT/data/" TILE_SIZE=256 HICT_WORKERS_TOTAL_MAX=256 java -jar ${SCRIPT_DIR}/build/libs/hict_server-*-fat.jar $@

scripts/run_optional_data_tests.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JVM_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+DATA_DIR="${DATA_DIR:-/mnt/Models/HiCT/data}"
+PORT="${PORT:-5011}"
+PROCESSED_DIR="${PROCESSED_DIR:-/tmp/hict_processed_cache_ci}"
+export GRADLE_USER_HOME="${GRADLE_USER_HOME:-/tmp/hict_gradle_home}"
+mkdir -p "${GRADLE_USER_HOME}"
+
+HICT_REL="build/quad/combined_ind2_4DN.hict.hdf5"
+FASTA_REL="build/quad/quad_combined_ind2.fasta"
+BW_REL="build/quad/ind2.coverage.bw"
+BED_REL="build/quad/ind2.alignments.bed.gz"
+GFF_REL=""
+for candidate in \
+  "build/quad/annotation_miniprot/ind2.miniprot.gff3" \
+  "build/quad/ind2.miniprot.gff3" \
+  "build/quad/ind2.features.gff3"
+do
+  if [[ -f "${DATA_DIR}/${candidate}" ]]; then
+    GFF_REL="${candidate}"
+    break
+  fi
+done
+
+HICT_ABS="${DATA_DIR}/${HICT_REL}"
+FASTA_ABS="${DATA_DIR}/${FASTA_REL}"
+BW_ABS="${DATA_DIR}/${BW_REL}"
+BED_ABS="${DATA_DIR}/${BED_REL}"
+GFF_ABS=""
+if [[ -n "${GFF_REL}" ]]; then
+  GFF_ABS="${DATA_DIR}/${GFF_REL}"
+fi
+
+cd "${JVM_DIR}"
+
+echo "[optional] Building fat jar"
+./gradlew shadowJar >/dev/null
+
+mkdir -p "${PROCESSED_DIR}"
+JAR_PATH="$(ls -1 build/libs/hict_server-*-fat.jar | head -n 1)"
+if [[ -z "${JAR_PATH}" ]]; then
+  echo "[optional] ERROR: fat jar was not produced"
+  exit 1
+fi
+
+if [[ ! -f "${HICT_ABS}" ]]; then
+  echo "[optional] Skip: ${HICT_ABS} does not exist"
+  exit 0
+fi
+
+echo "[optional] Starting API server on :${PORT}"
+VXPORT="${PORT}" SERVE_WEBUI=false DATA_DIR="${DATA_DIR}" PROCESSED_DIR="${PROCESSED_DIR}" \
+  java -jar "${JAR_PATH}" start-api-server >/tmp/hict_optional_tests_server.log 2>&1 &
+SERVER_PID=$!
+cleanup() {
+  if ps -p "${SERVER_PID}" >/dev/null 2>&1; then
+    kill "${SERVER_PID}" >/dev/null 2>&1 || true
+    wait "${SERVER_PID}" >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT
+
+for _ in $(seq 1 40); do
+  if curl -fsS "http://localhost:${PORT}/version" >/dev/null 2>&1; then
+    break
+  fi
+  sleep 1
+done
+curl -fsS "http://localhost:${PORT}/version" >/dev/null
+
+echo "[optional] Opening HiCT"
+curl -fsS -X POST "http://localhost:${PORT}/open" \
+  -H "content-type: application/json" \
+  -d "{\"filename\":\"${HICT_REL}\"}" >/dev/null
+
+echo "[optional] Opening Cooler weights track"
+curl -fsS -X POST "http://localhost:${PORT}/tracks/open_cooler_weights" \
+  -H "content-type: application/json" \
+  -d '{"name":"Cooler weights"}' >/dev/null
+
+COOLER_TRACK_ID="$(
+  curl -fsS -X POST "http://localhost:${PORT}/tracks/list" \
+    -H "content-type: application/json" \
+    -d '{}' | jq -r '.[] | select(.type=="COOLER_WEIGHTS") | .trackId' | head -n1
+)"
+if [[ -z "${COOLER_TRACK_ID}" ]]; then
+  echo "[optional] ERROR: Cooler weights track was not created"
+  exit 1
+fi
+curl -fsS -X POST "http://localhost:${PORT}/tracks/update" \
+  -H "content-type: application/json" \
+  -d "{\"trackId\":\"${COOLER_TRACK_ID}\",\"logScale\":true}" >/dev/null
+
+if [[ -f "${FASTA_ABS}" ]]; then
+  echo "[optional] Linking FASTA"
+  curl -fsS -X POST "http://localhost:${PORT}/link_fasta" \
+    -H "content-type: application/json" \
+    -d "{\"fastaFilename\":\"${FASTA_REL}\",\"allowMismatch\":true}" >/dev/null
+else
+  echo "[optional] FASTA not found, skipping FASTA link checks"
+fi
+
+if [[ -f "${BW_ABS}" ]]; then
+  echo "[optional] Opening BigWig track"
+  curl -fsS -X POST "http://localhost:${PORT}/tracks/open" \
+    -H "content-type: application/json" \
+    -d "{\"filename\":\"${BW_REL}\"}" >/dev/null
+fi
+
+if [[ -f "${BED_ABS}" ]]; then
+  echo "[optional] Opening BED track"
+  curl -fsS -X POST "http://localhost:${PORT}/tracks/open" \
+    -H "content-type: application/json" \
+    -d "{\"filename\":\"${BED_REL}\"}" >/dev/null
+fi
+
+if [[ -n "${GFF_ABS}" && -f "${GFF_ABS}" ]]; then
+  echo "[optional] Opening GFF track"
+  curl -fsS -X POST "http://localhost:${PORT}/tracks/open" \
+    -H "content-type: application/json" \
+    -d "{\"filename\":\"${GFF_REL}\"}" >/dev/null
+fi
+
+echo "[optional] Querying tracks in PIXELS/BINS/BP units"
+curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+  -H "content-type: application/json" \
+  -d '{"unit":"PIXELS","startPx":0,"endPx":6000,"widthPx":1200,"bpResolution":50000}' \
+  | jq -e '.tracks | length >= 0' >/dev/null
+curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+  -H "content-type: application/json" \
+  -d '{"unit":"BINS","startBin":0,"endBin":6000,"widthPx":1200,"bpResolution":50000}' \
+  | jq -e '.tracks | length >= 0' >/dev/null
+curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+  -H "content-type: application/json" \
+  -d '{"unit":"BP","startBP":0,"endBP":300000000,"widthPx":1200,"bpResolution":50000}' \
+  | jq -e '.tracks | length >= 0' >/dev/null
+
+echo "[optional] Querying numeric matrix in JSON and binary modes"
+MATRIX_JSON="$(curl -fsS -X POST "http://localhost:${PORT}/matrix/query" \
+  -H "content-type: application/json" \
+  -d '{"bpResolution":50000,"unit":"PIXELS","startRowPx":0,"endRowPx":32,"startColPx":0,"endColPx":32,"signalMode":"TRADITIONAL_NORMALIZED","format":"JSON"}')"
+echo "${MATRIX_JSON}" | jq -e '.rows == 32 and .cols == 32 and (.values|length)==1024' >/dev/null
+
+MATRIX_BIN_HEADERS="$(mktemp)"
+MATRIX_BIN_BODY="$(mktemp)"
+curl -fsS -D "${MATRIX_BIN_HEADERS}" -o "${MATRIX_BIN_BODY}" -X POST "http://localhost:${PORT}/matrix/query" \
+  -H "content-type: application/json" \
+  -d '{"bpResolution":50000,"unit":"PIXELS","startRowPx":0,"endRowPx":16,"startColPx":0,"endColPx":16,"signalMode":"COOLER_WEIGHTED","format":"BINARY_FLOAT32"}' >/dev/null
+grep -qi '^x-hict-rows: 16' "${MATRIX_BIN_HEADERS}"
+grep -qi '^x-hict-cols: 16' "${MATRIX_BIN_HEADERS}"
+grep -qi '^x-hict-dtype: float32' "${MATRIX_BIN_HEADERS}"
+[[ "$(wc -c < "${MATRIX_BIN_BODY}")" -eq $((16 * 16 * 4)) ]]
+rm -f "${MATRIX_BIN_HEADERS}" "${MATRIX_BIN_BODY}"
+
+echo "[optional] Cooler weights query sanity check"
+curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+  -H "content-type: application/json" \
+  -d '{"unit":"PIXELS","startPx":0,"endPx":7000,"widthPx":1536,"bpResolution":50000}' \
+  | jq -e '.tracks[] | select(.type=="COOLER_WEIGHTS") | (.bins | length) > 0' >/dev/null
+
+if [[ -f "${BW_ABS}" && -f "${BED_ABS}" ]]; then
+  echo "[optional] BED vs BigWig sanity check"
+  QUERY_JSON="$(curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+    -H "content-type: application/json" \
+    -d '{"unit":"PIXELS","startPx":0,"endPx":7000,"widthPx":1536,"bpResolution":50000}')"
+  BW_BINS="$(echo "${QUERY_JSON}" | jq -r '.tracks[] | select(.sourceFile? == null or true) | select(.type=="BIGWIG") | .bins | length' | head -n1 || true)"
+  BED_BINS="$(echo "${QUERY_JSON}" | jq -r '.tracks[] | select(.type=="BED") | .bins | length' | head -n1 || true)"
+  if [[ -n "${BW_BINS}" && -n "${BED_BINS}" ]]; then
+    if [[ "${BW_BINS}" -eq 0 || "${BED_BINS}" -eq 0 ]]; then
+      echo "[optional] ERROR: BED/BigWig bins are empty"
+      exit 1
+    fi
+  fi
+fi
+
+if [[ -n "${GFF_ABS}" && -f "${GFF_ABS}" ]]; then
+  echo "[optional] GFF/GTF structured bins sanity check"
+  GFF_QUERY_JSON="$(curl -fsS -X POST "http://localhost:${PORT}/tracks/query_1d" \
+    -H "content-type: application/json" \
+    -d '{"unit":"PIXELS","startPx":0,"endPx":7000,"widthPx":1536,"bpResolution":50000}')"
+  echo "${GFF_QUERY_JSON}" | jq -e '
+    .tracks[]
+    | select(.type=="GFF_GTF")
+    | (.bins | length) > 0
+  ' >/dev/null
+  echo "${GFF_QUERY_JSON}" | jq -e '
+    .tracks[]
+    | select(.type=="GFF_GTF")
+    | (.bins | any((.blocks // []) | length > 0))
+  ' >/dev/null
+  echo "${GFF_QUERY_JSON}" | jq -e '
+    .tracks[]
+    | select(.type=="GFF_GTF")
+    | (.bins | any(.strand == "+" or .strand == "-"))
+  ' >/dev/null
+fi
+
+echo "[optional] OK"

scripts/run_smoke_tests.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JVM_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+WEBUI_DIR="$(cd "${JVM_DIR}/../HiCT_WebUI" && pwd)"
+export GRADLE_USER_HOME="${GRADLE_USER_HOME:-/tmp/hict_gradle_home}"
+mkdir -p "${GRADLE_USER_HOME}"
+
+echo "[smoke] Building JVM module"
+cd "${JVM_DIR}"
+./gradlew compileJava
+echo "[smoke] Running JVM tests"
+./gradlew test
+
+if [[ -d "${WEBUI_DIR}" ]]; then
+  echo "[smoke] Type-checking and building WebUI"
+  cd "${WEBUI_DIR}"
+  npm run type-check
+  npm run build-only
+fi
+
+echo "[smoke] OK"