From 7031e9e84c3ce268d9ac983a3229273af5d75414 Mon Sep 17 00:00:00 2001 From: Alexander Serdyukov Date: Sat, 21 Mar 2026 01:09:40 +0400 Subject: [PATCH 1/5] Modern attempt to add support for 1D tracks rendering, still not working well --- build.gradle.kts | 5 + ldbg.sh | 2 +- .../hict_library/assembly/FASTAProcessor.java | 493 ++++++ .../ctlab/hict/hict_server/MainVerticle.java | 2 + .../assembly/ContigDescriptorDTO.java | 4 + .../response/fasta/FastaLinkResponseDTO.java | 96 ++ .../handlers/fileop/FileOpHandlersHolder.java | 119 ++ .../handlers/files/FSHandlersHolder.java | 16 +- .../handlers/tracks/TrackHandlersHolder.java | 169 +++ .../hict_server/tracks/Track1DManager.java | 1335 +++++++++++++++++ .../util/shareable/ShareableWrappers.java | 7 + version.txt | 2 +- 12 files changed, 2247 insertions(+), 3 deletions(-) create mode 100644 src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/fasta/FastaLinkResponseDTO.java create mode 100644 src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java create mode 100644 src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java diff --git a/build.gradle.kts b/build.gradle.kts index cec2196..3796363 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -41,6 +41,9 @@ repositories { maven { url = uri("https://maven.scijava.org/content/repositories/public/") } + maven { + url = uri("https://nexus.bioviz.org/repository/maven-releases/") + } } val vertxVersion = "4.4.1" @@ -132,6 +135,8 @@ dependencies { // https://mvnrepository.com/artifact/org.scijava/native-lib-loader implementation("org.scijava:native-lib-loader:2.4.0") implementation("info.picocli:picocli:4.7.6") + implementation("com.github.samtools:htsjdk:4.1.3") + implementation("org.broad.igv:bigwig:3.0.0") } diff --git a/ldbg.sh b/ldbg.sh index 713b2fc..e59fd1c 100755 --- a/ldbg.sh +++ b/ldbg.sh @@ -1,4 +1,4 @@ #!/bin/bash export VERTXWEB_ENVIRONMENT="dev" SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -VXPORT=5000 DATA_DIR="/mnt/Models/HiCT/data/" TILE_SIZE=256 java -jar ${SCRIPT_DIR}/build/libs/hict_server-*-fat.jar +VXPORT=5000 DATA_DIR="/mnt/Models/HiCT/data/" TILE_SIZE=256 java -jar ${SCRIPT_DIR}/build/libs/hict_server-*-fat.jar $@ diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_library/assembly/FASTAProcessor.java b/src/main/java/ru/itmo/ctlab/hict/hict_library/assembly/FASTAProcessor.java index bb8f6fe..e5434e6 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_library/assembly/FASTAProcessor.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_library/assembly/FASTAProcessor.java @@ -26,9 +26,502 @@ import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; +import ru.itmo.ctlab.hict.hict_library.domain.ContigDescriptor; +import ru.itmo.ctlab.hict.hict_library.domain.ScaffoldDescriptor; +import ru.itmo.ctlab.hict.hict_library.trees.ScaffoldTree; +import ru.itmo.ctlab.hict.hict_library.trees.ContigTree; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Comparator; +import java.util.zip.GZIPInputStream; @RequiredArgsConstructor public class FASTAProcessor { + private static final int MAX_MISMATCHES_IN_REPORT = 10; + private final @NotNull ChunkedFile chunkedFile; + + public @NotNull FASTALinkCompatibilityReport analyzeLinkCandidate(final @NotNull Path fastaPath) { + final var fastaRecords = readSequenceSummaries(fastaPath); + final var assemblyEntries = currentAssemblyEntries(); + + final boolean sameRecordCount = fastaRecords.size() == assemblyEntries.size(); + final int sharedCount = Math.min(fastaRecords.size(), assemblyEntries.size()); + + boolean sameOrderAndLength = sameRecordCount; + boolean sameOrderLengthAndCurrentNames = sameRecordCount; + boolean sameOrderLengthAndOriginalNames = sameRecordCount; + boolean sameOrderLengthAndSourceNames = sameRecordCount; + + final var mismatches = new ArrayList(); + for (int i = 0; i < sharedCount; ++i) { + final var fasta = fastaRecords.get(i); + final var assembly = assemblyEntries.get(i); + final boolean sameLength = fasta.lengthBp() == assembly.lengthBp(); + final boolean anyNameMatch = fasta.name().equals(assembly.currentName()) + || fasta.name().equals(assembly.originalName()) + || fasta.name().equals(assembly.sourceName()); + sameOrderAndLength &= sameLength; + sameOrderLengthAndCurrentNames &= sameLength && fasta.name().equals(assembly.currentName()); + sameOrderLengthAndOriginalNames &= sameLength && fasta.name().equals(assembly.originalName()); + sameOrderLengthAndSourceNames &= sameLength && fasta.name().equals(assembly.sourceName()); + if ((!sameLength || !anyNameMatch) + && mismatches.size() < MAX_MISMATCHES_IN_REPORT) { + mismatches.add(new FASTALinkCompatibilityReport.MismatchAtIndex( + i, + fasta.name(), + fasta.lengthBp(), + assembly.currentName(), + assembly.originalName(), + assembly.sourceName(), + assembly.lengthBp() + )); + } + } + + for (int i = sharedCount; i < fastaRecords.size() && mismatches.size() < MAX_MISMATCHES_IN_REPORT; ++i) { + final var fasta = fastaRecords.get(i); + mismatches.add(new FASTALinkCompatibilityReport.MismatchAtIndex( + i, + fasta.name(), + fasta.lengthBp(), + null, + null, + null, + -1L + )); + } + for (int i = sharedCount; i < assemblyEntries.size() && mismatches.size() < MAX_MISMATCHES_IN_REPORT; ++i) { + final var assembly = assemblyEntries.get(i); + mismatches.add(new FASTALinkCompatibilityReport.MismatchAtIndex( + i, + null, + -1L, + assembly.currentName(), + assembly.originalName(), + assembly.sourceName(), + assembly.lengthBp() + )); + } + + final boolean sameLengthMultiset = sameRecordCount + && fastaRecords.stream().map(FASTASequenceSummary::lengthBp).sorted().toList() + .equals(assemblyEntries.stream().map(AssemblyContigEntry::lengthBp).sorted().toList()); + + final var warnings = new ArrayList(); + if (!sameRecordCount) { + warnings.add(String.format( + Locale.ROOT, + "FASTA contains %d sequences, while the current Hi-C assembly contains %d contigs.", + fastaRecords.size(), + assemblyEntries.size() + )); + } + if (!sameOrderAndLength) { + if (sameLengthMultiset) { + warnings.add("FASTA sequence lengths match as a set, but the current Hi-C assembly order differs."); + } else { + warnings.add("FASTA sequence order and lengths do not match the current Hi-C assembly."); + } + } else if (!(sameOrderLengthAndCurrentNames || sameOrderLengthAndOriginalNames || sameOrderLengthAndSourceNames)) { + warnings.add("FASTA sequence lengths and order match the current Hi-C assembly, but contig names differ."); + } + + return new FASTALinkCompatibilityReport( + fastaPath.getFileName().toString(), + fastaRecords.size(), + assemblyEntries.size(), + sameRecordCount, + sameOrderAndLength, + sameOrderLengthAndCurrentNames, + sameOrderLengthAndOriginalNames, + sameOrderLengthAndSourceNames, + sameLengthMultiset, + warnings, + mismatches + ); + } + + public @NotNull Map buildSourceNameAliases(final @NotNull Path fastaPath) { + final var fastaRecords = readSequenceSummaries(fastaPath); + final var sourceDescriptors = this.chunkedFile.getOriginalDescriptors().values().stream() + .sorted(Comparator.comparingInt(ContigDescriptor::getContigId)) + .toList(); + if (fastaRecords.size() != sourceDescriptors.size()) { + return Map.of(); + } + for (int i = 0; i < fastaRecords.size(); ++i) { + if (fastaRecords.get(i).lengthBp() != sourceDescriptors.get(i).getLengthBp()) { + return Map.of(); + } + } + final var aliases = new HashMap(); + for (int i = 0; i < fastaRecords.size(); ++i) { + aliases.put(sourceDescriptors.get(i).getContigNameInSourceFASTA(), fastaRecords.get(i).name()); + } + return aliases; + } + + public @NotNull String exportAssembly(final @NotNull Path fastaPath) { + final var sequences = readSequenceContents(fastaPath); + final var records = new ArrayList(); + final var contigs = this.chunkedFile.getAssemblyInfo().contigs(); + final var scaffolds = this.chunkedFile.getAssemblyInfo().scaffolds(); + + int scaffoldIndex = 0; + long assemblyPosition = 0L; + final var currentUnscaffolded = new StringBuilder(); + String currentUnscaffoldedName = null; + final var currentScaffold = new StringBuilder(); + String currentScaffoldName = null; + long currentScaffoldEnd = -1L; + long currentSpacerLength = 0L; + boolean scaffoldHasContent = false; + + for (final var contig : contigs) { + while (scaffoldIndex < scaffolds.size() + && scaffolds.get(scaffoldIndex).scaffoldBordersBP().endBP() <= assemblyPosition) { + scaffoldIndex++; + } + + final var sequence = extractContigSequence(sequences, contig); + final ScaffoldTree.ScaffoldTuple coveringScaffold = + scaffoldIndex < scaffolds.size() + && scaffolds.get(scaffoldIndex).scaffoldDescriptor() != null + && scaffolds.get(scaffoldIndex).scaffoldBordersBP().startBP() <= assemblyPosition + && assemblyPosition < scaffolds.get(scaffoldIndex).scaffoldBordersBP().endBP() + ? scaffolds.get(scaffoldIndex) + : null; + + if (coveringScaffold != null) { + if (currentUnscaffoldedName != null && currentUnscaffolded.length() > 0) { + records.add(new FASTARecord(currentUnscaffoldedName, currentUnscaffolded.toString())); + currentUnscaffolded.setLength(0); + currentUnscaffoldedName = null; + } + final ScaffoldDescriptor scaffoldDescriptor = coveringScaffold.scaffoldDescriptor(); + final String scaffoldName = this.chunkedFile.getScaffoldDisplayName(scaffoldDescriptor.scaffoldId()); + if (!scaffoldName.equals(currentScaffoldName)) { + if (currentScaffoldName != null && currentScaffold.length() > 0) { + records.add(new FASTARecord(currentScaffoldName, currentScaffold.toString())); + currentScaffold.setLength(0); + } + currentScaffoldName = scaffoldName; + currentScaffoldEnd = coveringScaffold.scaffoldBordersBP().endBP(); + currentSpacerLength = scaffoldDescriptor.spacerLength(); + scaffoldHasContent = false; + } + if (scaffoldHasContent && currentSpacerLength > 0) { + currentScaffold.append("N".repeat((int) Math.min(Integer.MAX_VALUE, currentSpacerLength))); + } + currentScaffold.append(sequence); + scaffoldHasContent = true; + if (assemblyPosition + contig.descriptor().getLengthBp() >= currentScaffoldEnd) { + records.add(new FASTARecord(currentScaffoldName, currentScaffold.toString())); + currentScaffold.setLength(0); + currentScaffoldName = null; + currentScaffoldEnd = -1L; + currentSpacerLength = 0L; + scaffoldHasContent = false; + } + } else { + if (currentScaffoldName != null && currentScaffold.length() > 0) { + records.add(new FASTARecord(currentScaffoldName, currentScaffold.toString())); + currentScaffold.setLength(0); + currentScaffoldName = null; + currentScaffoldEnd = -1L; + currentSpacerLength = 0L; + scaffoldHasContent = false; + } + currentUnscaffoldedName = this.chunkedFile.getContigDisplayName(contig.descriptor().getContigId()); + currentUnscaffolded.setLength(0); + currentUnscaffolded.append(sequence); + records.add(new FASTARecord(currentUnscaffoldedName, currentUnscaffolded.toString())); + currentUnscaffolded.setLength(0); + currentUnscaffoldedName = null; + } + + assemblyPosition += contig.descriptor().getLengthBp(); + } + + if (currentScaffoldName != null && currentScaffold.length() > 0) { + records.add(new FASTARecord(currentScaffoldName, currentScaffold.toString())); + } + if (records.isEmpty()) { + throw new IllegalStateException("Current assembly does not contain any contigs"); + } + return renderRecords(records); + } + + public @NotNull String exportSelection(final @NotNull Path fastaPath, + final long fromBpX, + final long fromBpY, + final long toBpX, + final long toBpY) { + final var sequences = readSequenceContents(fastaPath); + final long selectionStart = Math.max(0L, Math.min(Math.min(fromBpX, fromBpY), Math.min(toBpX, toBpY))); + final long selectionEnd = Math.max(selectionStart + 1L, Math.max(Math.max(fromBpX, fromBpY), Math.max(toBpX, toBpY))); + final var builder = new StringBuilder(); + long assemblyPosition = 0L; + for (final var contig : this.chunkedFile.getAssemblyInfo().contigs()) { + final long contigStart = assemblyPosition; + final long contigEnd = assemblyPosition + contig.descriptor().getLengthBp(); + final long overlapStart = Math.max(selectionStart, contigStart); + final long overlapEnd = Math.min(selectionEnd, contigEnd); + if (overlapEnd > overlapStart) { + final long startInsideContig = overlapStart - contigStart; + final long endInsideContig = overlapEnd - contigStart; + builder.append(extractContigSlice(sequences, contig, startInsideContig, endInsideContig)); + } + assemblyPosition = contigEnd; + if (assemblyPosition >= selectionEnd) { + break; + } + } + if (builder.isEmpty()) { + throw new IllegalArgumentException("Selected region does not intersect the current assembly"); + } + return renderRecords(List.of(new FASTARecord( + String.format(Locale.ROOT, "selection_%d_%d", selectionStart, selectionEnd), + builder.toString() + ))); + } + + private @NotNull List currentAssemblyEntries() { + final var entries = new ArrayList(); + for (final ContigTree.ContigTuple tuple : this.chunkedFile.getAssemblyInfo().contigs()) { + final var descriptor = tuple.descriptor(); + entries.add(new AssemblyContigEntry( + this.chunkedFile.getContigDisplayName(descriptor.getContigId()), + this.chunkedFile.getContigOriginalName(descriptor.getContigId()), + descriptor.getContigNameInSourceFASTA(), + descriptor.getLengthBp() + )); + } + return entries; + } + + public static @NotNull List readSequenceSummaries(final @NotNull Path fastaPath) { + final var records = new ArrayList(); + try (final var stream = openPossiblyGzippedStream(fastaPath); + final var reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String currentName = null; + long currentLength = 0L; + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith(">")) { + if (currentName != null) { + records.add(new FASTASequenceSummary(currentName, currentLength)); + } + currentName = parseHeaderName(line); + currentLength = 0L; + continue; + } + if (currentName == null || line.isBlank()) { + continue; + } + currentLength += line.trim().length(); + } + if (currentName != null) { + records.add(new FASTASequenceSummary(currentName, currentLength)); + } + } catch (final IOException e) { + throw new RuntimeException("Failed to read FASTA file " + fastaPath, e); + } + if (records.isEmpty()) { + throw new IllegalArgumentException("FASTA file " + fastaPath.getFileName() + " does not contain any sequences"); + } + return records; + } + + public static @NotNull Map readSequenceContents(final @NotNull Path fastaPath) { + final var sequences = new HashMap(); + final var order = new ArrayList(); + try (final var stream = openPossiblyGzippedStream(fastaPath); + final var reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String currentName = null; + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith(">")) { + currentName = parseHeaderName(line); + if (!sequences.containsKey(currentName)) { + sequences.put(currentName, new StringBuilder()); + order.add(currentName); + } + continue; + } + if (currentName == null || line.isBlank()) { + continue; + } + sequences.get(currentName).append(line.trim()); + } + } catch (final IOException e) { + throw new RuntimeException("Failed to read FASTA file " + fastaPath, e); + } + final var result = new HashMap(); + for (final var name : order) { + result.put(name, sequences.get(name).toString()); + } + return result; + } + + private static @NotNull InputStream openPossiblyGzippedStream(final @NotNull Path fastaPath) throws IOException { + final var stream = Files.newInputStream(fastaPath); + final var name = fastaPath.getFileName().toString().toLowerCase(Locale.ROOT); + if (name.endsWith(".gz")) { + return new GZIPInputStream(stream); + } + return stream; + } + + private static @NotNull String parseHeaderName(final @NotNull String line) { + final var header = line.substring(1).trim(); + if (header.isEmpty()) { + throw new IllegalArgumentException("Encountered an empty FASTA header"); + } + final int firstWhitespace = findFirstWhitespace(header); + return (firstWhitespace >= 0 ? header.substring(0, firstWhitespace) : header).trim(); + } + + private static int findFirstWhitespace(final @NotNull String input) { + for (int i = 0; i < input.length(); ++i) { + if (Character.isWhitespace(input.charAt(i))) { + return i; + } + } + return -1; + } + + private @NotNull String extractContigSequence(final @NotNull Map sequences, + final @NotNull ContigTree.ContigTuple contig) { + return extractContigSlice(sequences, contig, 0L, contig.descriptor().getLengthBp()); + } + + private @NotNull String extractContigSlice(final @NotNull Map sequences, + final @NotNull ContigTree.ContigTuple contig, + final long startInsideContig, + final long endInsideContig) { + final var descriptor = contig.descriptor(); + final var source = sequences.get(descriptor.getContigNameInSourceFASTA()); + if (source == null) { + throw new IllegalArgumentException( + "Linked FASTA does not contain sequence '" + descriptor.getContigNameInSourceFASTA() + "'" + ); + } + final int sourceStart = Math.toIntExact(descriptor.getOffsetInSourceFASTA() + startInsideContig); + final int sourceEnd = Math.toIntExact(descriptor.getOffsetInSourceFASTA() + endInsideContig); + if (sourceStart < 0 || sourceEnd > source.length() || sourceStart >= sourceEnd) { + throw new IllegalArgumentException( + "Requested source FASTA slice [" + sourceStart + ", " + sourceEnd + ") is outside sequence '" + + descriptor.getContigNameInSourceFASTA() + "' of length " + source.length() + ); + } + final var subsequence = source.substring(sourceStart, sourceEnd); + return switch (contig.direction()) { + case FORWARD -> subsequence; + case REVERSED -> reverseComplement(subsequence); + }; + } + + private static @NotNull String reverseComplement(final @NotNull String sequence) { + final var builder = new StringBuilder(sequence.length()); + for (int i = sequence.length() - 1; i >= 0; --i) { + builder.append(complement(sequence.charAt(i))); + } + return builder.toString(); + } + + private static char complement(final char base) { + return switch (Character.toUpperCase(base)) { + case 'A' -> 'T'; + case 'T' -> 'A'; + case 'C' -> 'G'; + case 'G' -> 'C'; + case 'N' -> 'N'; + case 'R' -> 'Y'; + case 'Y' -> 'R'; + case 'S' -> 'S'; + case 'W' -> 'W'; + case 'K' -> 'M'; + case 'M' -> 'K'; + case 'B' -> 'V'; + case 'D' -> 'H'; + case 'H' -> 'D'; + case 'V' -> 'B'; + default -> 'N'; + }; + } + + private static @NotNull String renderRecords(final @NotNull List records) { + final var builder = new StringBuilder(); + for (final var record : records) { + builder.append('>').append(record.name()).append('\n'); + final var sequence = record.sequence(); + for (int i = 0; i < sequence.length(); i += 80) { + builder.append(sequence, i, Math.min(sequence.length(), i + 80)).append('\n'); + } + } + return builder.toString(); + } + + private record AssemblyContigEntry( + @NotNull String currentName, + @NotNull String originalName, + @NotNull String sourceName, + long lengthBp + ) { + } + + public record FASTASequenceSummary(@NotNull String name, long lengthBp) { + } + + private record FASTARecord(@NotNull String name, @NotNull String sequence) { + } + + public record FASTALinkCompatibilityReport( + @NotNull String fastaFilename, + int fastaRecordCount, + int assemblyContigCount, + boolean sameRecordCount, + boolean sameOrderAndLength, + boolean sameOrderLengthAndCurrentNames, + boolean sameOrderLengthAndOriginalNames, + boolean sameOrderLengthAndSourceNames, + boolean sameLengthMultiset, + @NotNull List<@NotNull String> warnings, + @NotNull List<@NotNull MismatchAtIndex> mismatches + ) { + public FASTALinkCompatibilityReport { + warnings = List.copyOf(warnings); + mismatches = List.copyOf(mismatches); + } + + public boolean hasWarnings() { + return !warnings.isEmpty(); + } + + public record MismatchAtIndex( + int index, + @Nullable String fastaName, + long fastaLengthBp, + @Nullable String assemblyCurrentName, + @Nullable String assemblyOriginalName, + @Nullable String assemblySourceName, + long assemblyLengthBp + ) { + } + } } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java index c6316f4..6190262 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java @@ -56,6 +56,7 @@ import ru.itmo.ctlab.hict.hict_server.handlers.conversion.ConversionHandlersHolder; import ru.itmo.ctlab.hict.hict_server.handlers.info.InfoHandlersHolder; import ru.itmo.ctlab.hict.hict_server.handlers.tiles.TileHandlersHolder; +import ru.itmo.ctlab.hict.hict_server.handlers.tracks.TrackHandlersHolder; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; import java.awt.*; @@ -186,6 +187,7 @@ public void start(final Promise startPromise) throws Exception { handlersHolders.add(new NameMappingHandlersHolder(vertx)); handlersHolders.add(new ConversionHandlersHolder(vertx)); handlersHolders.add(new InfoHandlersHolder(vertx)); + handlersHolders.add(new TrackHandlersHolder(vertx)); router.route().failureHandler(ctx -> { log.error("An exception was caught at router top-level", ctx.failure()); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/assembly/ContigDescriptorDTO.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/assembly/ContigDescriptorDTO.java index 106be21..9906db6 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/assembly/ContigDescriptorDTO.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/assembly/ContigDescriptorDTO.java @@ -36,6 +36,8 @@ public record ContigDescriptorDTO( int contigId, String contigName, String contigOriginalName, + String contigSourceName, + int contigOffsetInSource, int contigDirection, long contigLengthBp, Map contigLengthBins, @@ -48,6 +50,8 @@ public record ContigDescriptorDTO( ctg.descriptor().getContigId(), chunkedFile.getContigDisplayName(ctg.descriptor().getContigId()), chunkedFile.getContigOriginalName(ctg.descriptor().getContigId()), + ctg.descriptor().getContigNameInSourceFASTA(), + ctg.descriptor().getOffsetInSourceFASTA(), ctg.direction().ordinal(), ctg.descriptor().getLengthBp(), IntStream.range(1, resolutions.length).boxed().collect(Collectors.toMap(resIdx -> resolutions[resIdx], resIdx -> ctg.descriptor().getLengthBinsAtResolution()[resIdx])), diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/fasta/FastaLinkResponseDTO.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/fasta/FastaLinkResponseDTO.java new file mode 100644 index 0000000..6f01284 --- /dev/null +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/dto/response/fasta/FastaLinkResponseDTO.java @@ -0,0 +1,96 @@ +/* + * MIT License + * + * Copyright (c) 2021-2026. Aleksandr Serdiukov, Anton Zamyatin, Aleksandr Sinitsyn, Vitalii Dravgelis and Computer Technologies Laboratory ITMO University team. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package ru.itmo.ctlab.hict.hict_server.dto.response.fasta; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import ru.itmo.ctlab.hict.hict_library.assembly.FASTAProcessor; + +import java.util.List; + +public record FastaLinkResponseDTO( + @NotNull String fastaFilename, + boolean linked, + boolean requiresConfirmation, + @NotNull List<@NotNull String> warnings, + @NotNull CompatibilityDTO compatibility +) { + public static @NotNull FastaLinkResponseDTO fromReport(final @NotNull FASTAProcessor.FASTALinkCompatibilityReport report, + final boolean linked, + final boolean requiresConfirmation) { + return new FastaLinkResponseDTO( + report.fastaFilename(), + linked, + requiresConfirmation, + report.warnings(), + new CompatibilityDTO( + report.fastaRecordCount(), + report.assemblyContigCount(), + report.sameRecordCount(), + report.sameOrderAndLength(), + report.sameOrderLengthAndCurrentNames(), + report.sameOrderLengthAndOriginalNames(), + report.sameOrderLengthAndSourceNames(), + report.sameLengthMultiset(), + report.mismatches().stream().map(MismatchDTO::fromReport).toList() + ) + ); + } + + public record CompatibilityDTO( + int fastaRecordCount, + int assemblyContigCount, + boolean sameRecordCount, + boolean sameOrderAndLength, + boolean sameOrderLengthAndCurrentNames, + boolean sameOrderLengthAndOriginalNames, + boolean sameOrderLengthAndSourceNames, + boolean sameLengthMultiset, + @NotNull List<@NotNull MismatchDTO> mismatches + ) { + } + + public record MismatchDTO( + int index, + @Nullable String fastaName, + long fastaLengthBp, + @Nullable String assemblyCurrentName, + @Nullable String assemblyOriginalName, + @Nullable String assemblySourceName, + long assemblyLengthBp + ) { + public static @NotNull MismatchDTO fromReport(final @NotNull FASTAProcessor.FASTALinkCompatibilityReport.MismatchAtIndex mismatch) { + return new MismatchDTO( + mismatch.index(), + mismatch.fastaName(), + mismatch.fastaLengthBp(), + mismatch.assemblyCurrentName(), + mismatch.assemblyOriginalName(), + mismatch.assemblySourceName(), + mismatch.assemblyLengthBp() + ); + } + } +} diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java index 3df8ecc..48b63ba 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java @@ -39,8 +39,10 @@ import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; import ru.itmo.ctlab.hict.hict_server.dto.response.assembly.AssemblyInfoDTO; +import ru.itmo.ctlab.hict.hict_server.dto.response.fasta.FastaLinkResponseDTO; import ru.itmo.ctlab.hict.hict_server.dto.response.fileop.OpenFileResponseDTO; import ru.itmo.ctlab.hict.hict_server.handlers.util.TileStatisticHolder; +import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; import java.io.IOException; @@ -83,6 +85,13 @@ public void addHandlersToRouter(final @NotNull Router router) { } final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var oldTrackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (oldTrackManagerWrapper != null) { + oldTrackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); + oldTrackManagerWrapper.getTrack1DManager().close(); + } + map.remove("linkedFastaPath"); + map.remove("linkedFastaFilename"); final var progress = new io.vertx.core.json.JsonObject() .put("stage", "starting") @@ -110,6 +119,7 @@ public void addHandlersToRouter(final @NotNull Router router) { map.put("openedFilename", filename); map.put("TileStatisticHolder", TileStatisticHolder.newDefaultStatisticHolder(chunkedFile.getResolutions().length)); + map.put("Track1DManager", new ShareableWrappers.Track1DManagerWrapper(new Track1DManager(dataDirectory))); map.put("openProgress", new io.vertx.core.json.JsonObject() .put("stage", "done") @@ -153,6 +163,7 @@ public void addHandlersToRouter(final @NotNull Router router) { .end(Json.encode( new io.vertx.core.json.JsonObject() .put("filename", filename) + .put("fastaFilename", map.getOrDefault("linkedFastaFilename", "")) .put("openFileResponse", generateOpenFileResponse(chunkedFile)) )); }); @@ -170,11 +181,119 @@ public void addHandlersToRouter(final @NotNull Router router) { map.remove("chunkedFile"); map.remove("TileStatisticHolder"); map.remove("openedFilename"); + map.remove("linkedFastaPath"); + map.remove("linkedFastaFilename"); + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.remove("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); + trackManagerWrapper.getTrack1DManager().close(); + } ctx.response() .putHeader("content-type", "application/json") .end(Json.encode(new io.vertx.core.json.JsonObject().put("status", "closed"))); }); + router.post("/link_fasta").blockingHandler(ctx -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + ctx.fail(new IllegalStateException("Open a Hi-C file before linking FASTA")); + return; + } + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("dataDirectory"); + if (dataDirectoryWrapper == null) { + ctx.fail(new RuntimeException("Data directory is not present in local map")); + return; + } + final var requestJSON = ctx.body().asJsonObject(); + final var fastaFilename = requestJSON.getString("fastaFilename"); + final boolean allowMismatch = requestJSON.getBoolean("allowMismatch", false); + if (fastaFilename == null || fastaFilename.isBlank()) { + ctx.fail(new IllegalArgumentException("FASTA filename is required")); + return; + } + + final Path fastaPath = dataDirectoryWrapper.getPath().resolve(fastaFilename).normalize().toAbsolutePath(); + if (!fastaPath.startsWith(dataDirectoryWrapper.getPath())) { + ctx.fail(new IllegalArgumentException("FASTA path " + fastaFilename + " is outside DATA_DIR")); + return; + } + if (!Files.exists(fastaPath) || !Files.isRegularFile(fastaPath)) { + ctx.fail(new IllegalArgumentException("FASTA file " + fastaFilename + " does not exist")); + return; + } + + final var report = chunkedFileWrapper.getChunkedFile().getFastaProcessor().analyzeLinkCandidate(fastaPath); + final boolean requiresConfirmation = report.hasWarnings() && !allowMismatch; + if (!requiresConfirmation) { + map.put("linkedFastaPath", new ShareableWrappers.PathWrapper(fastaPath)); + map.put("linkedFastaFilename", fastaFilename); + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource( + chunkedFileWrapper.getChunkedFile().getFastaProcessor().buildSourceNameAliases(fastaPath) + ); + } + } + + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(FastaLinkResponseDTO.fromReport(report, !requiresConfirmation, requiresConfirmation))); + }); + + router.post("/get_fasta_for_assembly").blockingHandler(ctx -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + ctx.fail(new IllegalStateException("Open a Hi-C file before exporting FASTA")); + return; + } + final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); + if (fastaPathWrapper == null) { + ctx.fail(new IllegalStateException("Link a FASTA file before exporting FASTA")); + return; + } + final var fasta = chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportAssembly(fastaPathWrapper.getPath()); + ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())); + }); + + router.post("/get_fasta_for_selection").blockingHandler(ctx -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + ctx.fail(new IllegalStateException("Open a Hi-C file before exporting FASTA")); + return; + } + final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); + if (fastaPathWrapper == null) { + ctx.fail(new IllegalStateException("Link a FASTA file before exporting FASTA")); + return; + } + final var requestJSON = ctx.body().asJsonObject(); + final var fromBpX = requestJSON.getLong("fromBpX"); + final var fromBpY = requestJSON.getLong("fromBpY"); + final var toBpX = requestJSON.getLong("toBpX"); + final var toBpY = requestJSON.getLong("toBpY"); + if (fromBpX == null || fromBpY == null || toBpX == null || toBpY == null) { + ctx.fail(new IllegalArgumentException("Selection coordinates must be provided")); + return; + } + final var fasta = chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportSelection( + fastaPathWrapper.getPath(), + fromBpX, + fromBpY, + toBpX, + toBpY + ); + ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())); + }); + router.post("/get_agp_for_assembly").blockingHandler(ctx -> { final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); log.debug("Got map"); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java index fde5a9f..fbfbcbb 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java @@ -42,6 +42,10 @@ @Slf4j public class FSHandlersHolder extends HandlersHolder { private final Vertx vertx; + private static final List FASTA_SUFFIXES = List.of( + ".fasta", ".fa", ".fna", ".fas", + ".fasta.gz", ".fa.gz", ".fna.gz", ".fas.gz" + ); @Override public void addHandlersToRouter(final @NotNull Router router) { @@ -89,7 +93,12 @@ public void addHandlersToRouter(final @NotNull Router router) { final List files; try (final var fileStream = Files.walk(dataDirectory)) { - files = fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).filter(p -> p.toLowerCase().endsWith(".fasta")).collect(Collectors.toList()); + files = fileStream + .filter(Files::isRegularFile) + .map(dataDirectory::relativize) + .map(Object::toString) + .filter(FSHandlersHolder::isFastaFilename) + .collect(Collectors.toList()); } catch (final IOException e) { throw new RuntimeException(e); } @@ -113,4 +122,9 @@ public void addHandlersToRouter(final @NotNull Router router) { ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)); }); } + + private static boolean isFastaFilename(final @NotNull String path) { + final var lowered = path.toLowerCase(); + return FASTA_SUFFIXES.stream().anyMatch(lowered::endsWith); + } } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java new file mode 100644 index 0000000..2b06eef --- /dev/null +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java @@ -0,0 +1,169 @@ +/* + * MIT License + * + * Copyright (c) 2021-2026. Aleksandr Serdiukov, Anton Zamyatin, Aleksandr Sinitsyn, Vitalii Dravgelis and Computer Technologies Laboratory ITMO University team. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package ru.itmo.ctlab.hict.hict_server.handlers.tracks; + +import io.vertx.core.Vertx; +import io.vertx.core.json.Json; +import io.vertx.core.shareddata.LocalMap; +import io.vertx.ext.web.Router; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; +import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; +import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; +import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; + +import java.util.Map; + +@RequiredArgsConstructor +public class TrackHandlersHolder extends HandlersHolder { + private final Vertx vertx; + + @Override + public void addHandlersToRouter(final @NotNull Router router) { + router.post("/tracks/list_files").blockingHandler(ctx -> { + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(manager.listTrackFiles())); + }); + + router.post("/tracks/open").blockingHandler(ctx -> { + final var request = ctx.body().asJsonObject(); + final var filename = request.getString("filename"); + if (filename == null || filename.isBlank()) { + ctx.fail(new IllegalArgumentException("Track filename is required")); + return; + } + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + final var summary = manager.openTrack( + filename, + request.getString("name"), + request.getString("color") + ); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(summary)); + }); + + router.post("/tracks/list").blockingHandler(ctx -> { + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(manager.listTracks())); + }); + + router.post("/tracks/update").blockingHandler(ctx -> { + final var request = ctx.body().asJsonObject(); + final var trackId = request.getString("trackId"); + if (trackId == null || trackId.isBlank()) { + ctx.fail(new IllegalArgumentException("trackId is required")); + return; + } + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + final var updated = manager.updateTrack( + trackId, + request.containsKey("visible") ? request.getBoolean("visible") : null, + request.getString("color"), + request.getString("name"), + request.getString("renderMode"), + request.getString("aggregationMode") + ); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(updated)); + }); + + router.post("/tracks/remove").blockingHandler(ctx -> { + final var request = ctx.body().asJsonObject(); + final var trackId = request.getString("trackId"); + if (trackId == null || trackId.isBlank()) { + ctx.fail(new IllegalArgumentException("trackId is required")); + return; + } + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + manager.removeTrack(trackId); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(Map.of("status", "removed", "trackId", trackId))); + }); + + router.post("/tracks/query_1d").blockingHandler(ctx -> { + final var request = ctx.body().asJsonObject(); + final var startBp = request.getLong("startBp", 0L); + final var endBp = request.getLong("endBp", startBp + 1L); + final var widthPx = request.getInteger("widthPx", 512); + + final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + final var chunkedFile = extractChunkedFile(map, ctx); + if (chunkedFile == null) { + return; + } + final var result = manager.queryVisibleTracks(chunkedFile, startBp, endBp, widthPx); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(result)); + }); + } + + private Track1DManager getTrackManager(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var managerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (managerWrapper == null) { + ctx.fail(new IllegalStateException("Track manager is not initialized. Open a HiCT file first.")); + return null; + } + return managerWrapper.getTrack1DManager(); + } + + private ChunkedFile extractChunkedFile(final @NotNull LocalMap map, final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); + return null; + } + return chunkedFileWrapper.getChunkedFile(); + } +} diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java new file mode 100644 index 0000000..20b5745 --- /dev/null +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java @@ -0,0 +1,1335 @@ +/* + * MIT License + * + * Copyright (c) 2021-2026. Aleksandr Serdiukov, Anton Zamyatin, Aleksandr Sinitsyn, Vitalii Dravgelis and Computer Technologies Laboratory ITMO University team. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package ru.itmo.ctlab.hict.hict_server.tracks; + +import htsjdk.samtools.SAMRecord; +import htsjdk.samtools.SAMSequenceDictionary; +import htsjdk.samtools.SamReader; +import htsjdk.samtools.SamReaderFactory; +import htsjdk.samtools.ValidationStringency; +import htsjdk.samtools.util.CloseableIterator; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.broad.igv.bbfile.BBFileReader; +import org.broad.igv.bbfile.BigWigIterator; +import org.broad.igv.bbfile.WigItem; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import ru.itmo.ctlab.hict.hict_library.assembly.FASTAProcessor; +import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; +import ru.itmo.ctlab.hict.hict_library.domain.ContigDirection; +import ru.itmo.ctlab.hict.hict_library.trees.ContigTree; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.IntStream; +import java.util.zip.GZIPInputStream; + +@Slf4j +public class Track1DManager { + private static final Set SUPPORTED_EXTENSIONS = Set.of( + ".bed", ".bed.gz", ".vcf", ".vcf.gz", ".bw", ".bigwig", ".bam" + ); + private static final List COLOR_PALETTE = List.of( + "#4e79a7", "#f28e2b", "#e15759", "#76b7b2", "#59a14f", + "#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ab" + ); + private static final int MAX_FEATURES_PER_QUERY = 250_000; + + private final @NotNull Path dataDirectory; + private final @NotNull ReadWriteLock lock = new ReentrantReadWriteLock(); + private final @NotNull LinkedHashMap tracks = new LinkedHashMap<>(); + private final @NotNull AtomicLong trackCounter = new AtomicLong(0L); + private volatile @NotNull Map linkedFastaAliasesBySource = Map.of(); + + public Track1DManager(final @NotNull Path dataDirectory) { + this.dataDirectory = dataDirectory.normalize().toAbsolutePath(); + } + + public @NotNull List listTrackFiles() { + try (final var stream = Files.walk(this.dataDirectory)) { + return stream + .filter(Files::isRegularFile) + .map(this.dataDirectory::relativize) + .map(Path::toString) + .filter(this::isSupportedTrackPath) + .sorted() + .toList(); + } catch (final IOException e) { + throw new RuntimeException("Failed to list track files", e); + } + } + + public @NotNull TrackSummary openTrack(final @NotNull String relativeFilename, + final String requestedName, + final String requestedColor) { + final var resolvedPath = resolveDataPath(relativeFilename); + final var trackType = TrackType.fromPath(resolvedPath); + if (trackType == TrackType.UNSUPPORTED) { + throw new IllegalArgumentException( + "Unsupported track format for " + relativeFilename + ". Supported: BED/VCF/BigWig/BAM." + ); + } + final var trackId = "trk_" + this.trackCounter.incrementAndGet(); + final var resolvedName = (requestedName == null || requestedName.isBlank()) + ? resolvedPath.getFileName().toString() + : requestedName.trim(); + final var color = normalizeColor(requestedColor, colorForIndex((int) this.trackCounter.get() - 1)); + final TrackDataSource dataSource = switch (trackType) { + case BED -> InMemoryTrackDataSource.fromBed(resolvedPath); + case VCF -> InMemoryTrackDataSource.fromVcf(resolvedPath); + case BIGWIG -> new BigWigTrackDataSource(resolvedPath); + case BAM -> new BamTrackDataSource(resolvedPath); + case UNSUPPORTED -> throw new IllegalStateException("Unexpected unsupported track type"); + }; + final var state = new TrackState( + trackId, + resolvedName, + trackType, + relativeFilename, + color, + true, + dataSource, + BamRenderMode.COVERAGE, + BigWigAggregationMode.MAX + ); + try { + this.lock.writeLock().lock(); + this.tracks.put(trackId, state); + } catch (final RuntimeException ex) { + closeDataSourceQuietly(dataSource); + throw ex; + } finally { + this.lock.writeLock().unlock(); + } + return state.toSummary(); + } + + public @NotNull List listTracks() { + try { + this.lock.readLock().lock(); + return this.tracks.values().stream().map(TrackState::toSummary).toList(); + } finally { + this.lock.readLock().unlock(); + } + } + + public @NotNull TrackSummary updateTrack(final @NotNull String trackId, + final Boolean visible, + final String color, + final String name, + final String renderMode, + final String aggregationMode) { + try { + this.lock.writeLock().lock(); + final var current = this.tracks.get(trackId); + if (current == null) { + throw new IllegalArgumentException("Unknown track id " + trackId); + } + final var updated = current.withUpdated( + visible == null ? current.visible : visible, + (color == null || color.isBlank()) ? current.color : normalizeColor(color, current.color), + (name == null || name.isBlank()) ? current.name : name.trim(), + parseBamRenderMode(renderMode, current.bamRenderMode()), + parseBigWigAggregationMode(aggregationMode, current.bigWigAggregationMode()) + ); + this.tracks.put(trackId, updated); + return updated.toSummary(); + } finally { + this.lock.writeLock().unlock(); + } + } + + public void removeTrack(final @NotNull String trackId) { + try { + this.lock.writeLock().lock(); + final var removed = this.tracks.remove(trackId); + if (removed != null) { + closeDataSourceQuietly(removed.dataSource()); + } + } finally { + this.lock.writeLock().unlock(); + } + } + + public void close() { + try { + this.lock.writeLock().lock(); + this.tracks.values().forEach(track -> closeDataSourceQuietly(track.dataSource())); + this.tracks.clear(); + } finally { + this.lock.writeLock().unlock(); + } + } + + public void setLinkedFastaAliasesBySource(final @Nullable Map aliases) { + this.linkedFastaAliasesBySource = aliases == null ? Map.of() : Map.copyOf(aliases); + } + + public @NotNull QueryResult queryVisibleTracks(final @NotNull ChunkedFile chunkedFile, + final long startBp, + final long endBp, + final int widthPx) { + final var queryStart = Math.max(0L, Math.min(startBp, endBp)); + final var queryEnd = Math.max(queryStart + 1L, Math.max(startBp, endBp)); + final var safeWidth = Math.max(1, widthPx); + final Map> sourceToAssemblySegments = + buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource); + final List trackRenders = new ArrayList<>(); + try { + this.lock.readLock().lock(); + this.tracks.values().stream() + .filter(track -> track.visible) + .forEach(track -> { + try { + trackRenders.add(track.query(sourceToAssemblySegments, queryStart, queryEnd, safeWidth)); + } catch (final RuntimeException ex) { + final var message = ex.getMessage() != null ? ex.getMessage() : ex.getClass().getSimpleName(); + log.error("Failed to query 1D track {} ({})", track.name(), track.trackId(), ex); + trackRenders.add(track.toErrorRender(message)); + } + }); + } finally { + this.lock.readLock().unlock(); + } + return new QueryResult(queryStart, queryEnd, safeWidth, trackRenders); + } + + private @NotNull Map> buildSourceToAssemblySegments(final @NotNull ChunkedFile chunkedFile, + final @NotNull Map linkedFastaAliasesBySource) { + final var sourceToAssemblySegments = new HashMap>(); + final var contigs = chunkedFile.getAssemblyInfo().contigs(); + long assemblyCursor = 0L; + for (int contigIndex = 0; contigIndex < contigs.size(); ++contigIndex) { + final ContigTree.ContigTuple tuple = contigs.get(contigIndex); + final var descriptor = tuple.descriptor(); + final var sourceName = descriptor.getContigNameInSourceFASTA(); + final var sourceStart = descriptor.getOffsetInSourceFASTA(); + final var sourceEnd = sourceStart + descriptor.getLengthBp(); + final var assemblyStart = assemblyCursor; + final var assemblyEnd = assemblyCursor + descriptor.getLengthBp(); + sourceToAssemblySegments.computeIfAbsent(sourceName, key -> new ArrayList<>()) + .add(new AssemblySegment(sourceStart, sourceEnd, assemblyStart, assemblyEnd, tuple.direction() == ContigDirection.REVERSED)); + final var aliasName = linkedFastaAliasesBySource.get(sourceName); + if (aliasName != null && !aliasName.equals(sourceName)) { + sourceToAssemblySegments.computeIfAbsent(aliasName, key -> new ArrayList<>()) + .add(new AssemblySegment(sourceStart, sourceEnd, assemblyStart, assemblyEnd, tuple.direction() == ContigDirection.REVERSED)); + } + assemblyCursor = assemblyEnd; + } + sourceToAssemblySegments.values().forEach(list -> list.sort(Comparator.comparingLong(AssemblySegment::sourceStart))); + return sourceToAssemblySegments; + } + + private boolean isSupportedTrackPath(final @NotNull String path) { + final var lowered = path.toLowerCase(Locale.ROOT); + return SUPPORTED_EXTENSIONS.stream().anyMatch(lowered::endsWith); + } + + private @NotNull Path resolveDataPath(final @NotNull String relativePath) { + final var resolved = this.dataDirectory.resolve(relativePath).normalize().toAbsolutePath(); + if (!resolved.startsWith(this.dataDirectory)) { + throw new IllegalArgumentException("Path " + relativePath + " is outside DATA_DIR"); + } + if (!Files.exists(resolved) || !Files.isRegularFile(resolved)) { + throw new IllegalArgumentException("Track file " + relativePath + " does not exist"); + } + return resolved; + } + + private static @NotNull String normalizeColor(final String requestedColor, final @NotNull String fallback) { + if (requestedColor == null) { + return fallback; + } + final var trimmed = requestedColor.trim(); + if (trimmed.matches("^#[0-9a-fA-F]{6}$")) { + return trimmed.toLowerCase(Locale.ROOT); + } + return fallback; + } + + private static @NotNull String colorForIndex(final int index) { + if (COLOR_PALETTE.isEmpty()) { + return "#4e79a7"; + } + return COLOR_PALETTE.get(Math.floorMod(index, COLOR_PALETTE.size())); + } + + private static @NotNull BamRenderMode parseBamRenderMode(final String mode, final @NotNull BamRenderMode fallback) { + if (mode == null || mode.isBlank()) { + return fallback; + } + try { + return BamRenderMode.valueOf(mode.trim().toUpperCase(Locale.ROOT)); + } catch (final IllegalArgumentException ex) { + throw new IllegalArgumentException("Unsupported BAM render mode: " + mode + ". Supported: COVERAGE, READ_DENSITY"); + } + } + + private static @NotNull BigWigAggregationMode parseBigWigAggregationMode(final String mode, + final @NotNull BigWigAggregationMode fallback) { + if (mode == null || mode.isBlank()) { + return fallback; + } + try { + return BigWigAggregationMode.valueOf(mode.trim().toUpperCase(Locale.ROOT)); + } catch (final IllegalArgumentException ex) { + throw new IllegalArgumentException("Unsupported BigWig aggregation mode: " + mode + ". Supported: MAX, MEAN, SUM"); + } + } + + private static void closeDataSourceQuietly(final @NotNull TrackDataSource dataSource) { + try { + dataSource.close(); + } catch (final Exception e) { + log.warn("Failed to close track data source", e); + } + } + + private static long parseLongOrThrow(final @NotNull String value, final @NotNull String fieldName, final long lineNo) { + try { + return Long.parseLong(value); + } catch (final NumberFormatException nfe) { + throw new IllegalArgumentException("Cannot parse " + fieldName + " at line " + lineNo + ": " + value); + } + } + + private static double parseOptionalDouble(final String value, final double defaultValue) { + if (value == null || value.isBlank()) { + return defaultValue; + } + try { + return Double.parseDouble(value); + } catch (final NumberFormatException ignored) { + return defaultValue; + } + } + + private static @NotNull BufferedReader openMaybeGzipReader(final @NotNull Path filePath) throws IOException { + final InputStream baseStream = Files.newInputStream(filePath); + final InputStream dataStream; + final var lowered = filePath.getFileName().toString().toLowerCase(Locale.ROOT); + if (lowered.endsWith(".gz")) { + dataStream = new GZIPInputStream(baseStream); + } else { + dataStream = baseStream; + } + return new BufferedReader(new InputStreamReader(dataStream, StandardCharsets.UTF_8)); + } + + private static @NotNull Optional projectSourceIntervalOnSegment(final @NotNull AssemblySegment segment, + final long sourceStart, + final long sourceEnd, + final double value, + final String label, + final long queryStart, + final long queryEnd) { + final var clippedSourceStart = Math.max(sourceStart, segment.sourceStart()); + final var clippedSourceEnd = Math.min(sourceEnd, segment.sourceEnd()); + if (clippedSourceEnd <= clippedSourceStart) { + return Optional.empty(); + } + final var segmentLength = segment.sourceEnd() - segment.sourceStart(); + final var localStart = clippedSourceStart - segment.sourceStart(); + final var localEnd = clippedSourceEnd - segment.sourceStart(); + + final long assemblyStart; + final long assemblyEnd; + if (!segment.reversed()) { + assemblyStart = segment.assemblyStart() + localStart; + assemblyEnd = segment.assemblyStart() + localEnd; + } else { + assemblyStart = segment.assemblyStart() + (segmentLength - localEnd); + assemblyEnd = segment.assemblyStart() + (segmentLength - localStart); + } + + final var clippedAssemblyStart = Math.max(queryStart, Math.min(assemblyStart, assemblyEnd)); + final var clippedAssemblyEnd = Math.min(queryEnd, Math.max(assemblyStart, assemblyEnd)); + if (clippedAssemblyEnd <= clippedAssemblyStart) { + return Optional.empty(); + } + return Optional.of(new ProjectedFeature(clippedAssemblyStart, clippedAssemblyEnd, Math.max(0.0d, value), label)); + } + + private static @NotNull Optional mapAssemblyIntervalToSegmentSource(final @NotNull AssemblySegment segment, + final long assemblyStart, + final long assemblyEnd) { + final var overlapAssemblyStart = Math.max(assemblyStart, segment.assemblyStart()); + final var overlapAssemblyEnd = Math.min(assemblyEnd, segment.assemblyEnd()); + if (overlapAssemblyEnd <= overlapAssemblyStart) { + return Optional.empty(); + } + final var localStart = overlapAssemblyStart - segment.assemblyStart(); + final var localEnd = overlapAssemblyEnd - segment.assemblyStart(); + final var segmentLength = segment.sourceEnd() - segment.sourceStart(); + if (!segment.reversed()) { + return Optional.of(new SourceInterval(segment.sourceStart() + localStart, segment.sourceStart() + localEnd)); + } + final var sourceStart = segment.sourceStart() + (segmentLength - localEnd); + final var sourceEnd = segment.sourceStart() + (segmentLength - localStart); + return Optional.of(new SourceInterval(sourceStart, sourceEnd)); + } + + private static @NotNull List aggregateFeatures(final @NotNull List projectedFeatures, + final long queryStart, + final long queryEnd, + final int widthPx) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] maxValues = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + Arrays.fill(maxValues, 0.0d); + for (final var feature : projectedFeatures) { + int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); + int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + left = Math.max(0, Math.min(left, bucketCount - 1)); + right = Math.max(0, Math.min(right, bucketCount - 1)); + for (int i = left; i <= right; i++) { + maxValues[i] = Math.max(maxValues[i], feature.value()); + counts[i]++; + } + } + final var bins = new ArrayList(bucketCount); + for (int i = 0; i < bucketCount; i++) { + if (counts[i] <= 0) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + bins.add(new TrackBin(start, Math.max(start + 1, end), maxValues[i], counts[i], null)); + } + return bins; + } + + private static @NotNull List aggregateBigWigFeatures(final @NotNull List projectedFeatures, + final long queryStart, + final long queryEnd, + final int widthPx, + final @NotNull BigWigAggregationMode mode) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] maxValues = new double[bucketCount]; + final double[] weightedSums = new double[bucketCount]; + final double[] overlapSums = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + Arrays.fill(maxValues, 0.0d); + for (final var feature : projectedFeatures) { + int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); + int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + left = Math.max(0, Math.min(left, bucketCount - 1)); + right = Math.max(0, Math.min(right, bucketCount - 1)); + for (int i = left; i <= right; i++) { + final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(feature.end(), bucketEnd) - Math.max(feature.start(), bucketStart); + if (overlap <= 0L) { + continue; + } + maxValues[i] = Math.max(maxValues[i], feature.value()); + weightedSums[i] += feature.value() * overlap; + overlapSums[i] += overlap; + counts[i]++; + } + } + final var bins = new ArrayList(bucketCount); + for (int i = 0; i < bucketCount; i++) { + if (counts[i] <= 0) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var bucketWidth = Math.max(1.0d, end - start); + final double value = switch (mode) { + case MAX -> maxValues[i]; + case MEAN -> overlapSums[i] > 0.0d ? weightedSums[i] / overlapSums[i] : 0.0d; + case SUM -> weightedSums[i] / bucketWidth; + }; + bins.add(new TrackBin(start, Math.max(start + 1, end), value, counts[i], null)); + } + return bins; + } + + private static @NotNull List aggregateCoverageFeatures(final @NotNull List projectedFeatures, + final long queryStart, + final long queryEnd, + final int widthPx) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] coverage = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + Arrays.fill(coverage, 0.0d); + for (final var feature : projectedFeatures) { + int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); + int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + left = Math.max(0, Math.min(left, bucketCount - 1)); + right = Math.max(0, Math.min(right, bucketCount - 1)); + for (int i = left; i <= right; i++) { + final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(feature.end(), bucketEnd) - Math.max(feature.start(), bucketStart); + if (overlap <= 0L) { + continue; + } + final var norm = overlap / Math.max(1.0d, (double) (bucketEnd - bucketStart)); + coverage[i] += norm; + counts[i]++; + } + } + final var bins = new ArrayList(bucketCount); + for (int i = 0; i < bucketCount; i++) { + if (counts[i] <= 0) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + bins.add(new TrackBin(start, Math.max(start + 1, end), coverage[i], counts[i], null)); + } + return bins; + } + + private static @NotNull List aggregateReadDensityFeatures(final @NotNull List projectedFeatures, + final long queryStart, + final long queryEnd, + final int widthPx) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] values = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + Arrays.fill(values, 0.0d); + for (final var feature : projectedFeatures) { + final var center = feature.start() + ((feature.end() - feature.start()) >>> 1); + int idx = (int) Math.floor((center - queryStart) / bucketSpan); + idx = Math.max(0, Math.min(idx, bucketCount - 1)); + values[idx] += 1.0d; + counts[idx] += 1L; + } + final var bins = new ArrayList(bucketCount); + for (int i = 0; i < bucketCount; i++) { + if (counts[i] <= 0L) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + bins.add(new TrackBin(start, Math.max(start + 1, end), values[i], counts[i], null)); + } + return bins; + } + + private static void accumulateBigWigValue(final long featureStart, + final long featureEnd, + final double featureValue, + final long queryStart, + final long queryEnd, + final double bucketSpan, + final double[] maxValues, + final double[] weightedSums, + final double[] overlapSums, + final long[] counts) { + int left = (int) Math.floor((featureStart - queryStart) / bucketSpan); + int right = (int) Math.ceil((featureEnd - queryStart) / bucketSpan) - 1; + left = Math.max(0, Math.min(left, counts.length - 1)); + right = Math.max(0, Math.min(right, counts.length - 1)); + for (int i = left; i <= right; i++) { + final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(featureEnd, bucketEnd) - Math.max(featureStart, bucketStart); + if (overlap <= 0L) { + continue; + } + maxValues[i] = Math.max(maxValues[i], featureValue); + weightedSums[i] += featureValue * overlap; + overlapSums[i] += overlap; + counts[i]++; + } + } + + private static void accumulateCoverageValue(final long featureStart, + final long featureEnd, + final long queryStart, + final long queryEnd, + final double bucketSpan, + final double[] coverage, + final long[] counts) { + int left = (int) Math.floor((featureStart - queryStart) / bucketSpan); + int right = (int) Math.ceil((featureEnd - queryStart) / bucketSpan) - 1; + left = Math.max(0, Math.min(left, counts.length - 1)); + right = Math.max(0, Math.min(right, counts.length - 1)); + for (int i = left; i <= right; i++) { + final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(featureEnd, bucketEnd) - Math.max(featureStart, bucketStart); + if (overlap <= 0L) { + continue; + } + coverage[i] += overlap / Math.max(1.0d, (double) (bucketEnd - bucketStart)); + counts[i]++; + } + } + + private static void accumulateReadDensityValue(final long featureStart, + final long featureEnd, + final long queryStart, + final double bucketSpan, + final double[] values, + final long[] counts) { + final var center = featureStart + ((featureEnd - featureStart) >>> 1); + int idx = (int) Math.floor((center - queryStart) / bucketSpan); + idx = Math.max(0, Math.min(idx, counts.length - 1)); + values[idx] += 1.0d; + counts[idx] += 1L; + } + + private static @NotNull List finalizeBigWigBins(final long queryStart, + final long queryEnd, + final double bucketSpan, + final double[] maxValues, + final double[] weightedSums, + final double[] overlapSums, + final long[] counts, + final @NotNull BigWigAggregationMode mode) { + final var bins = new ArrayList(counts.length); + for (int i = 0; i < counts.length; i++) { + if (counts[i] <= 0L) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var bucketWidth = Math.max(1.0d, end - start); + final double value = switch (mode) { + case MAX -> maxValues[i]; + case MEAN -> overlapSums[i] > 0.0d ? weightedSums[i] / overlapSums[i] : 0.0d; + case SUM -> weightedSums[i] / bucketWidth; + }; + bins.add(new TrackBin(start, Math.max(start + 1, end), value, counts[i], null)); + } + return bins; + } + + private static @NotNull List finalizeBins(final long queryStart, + final long queryEnd, + final double bucketSpan, + final double[] values, + final long[] counts) { + final var bins = new ArrayList(counts.length); + for (int i = 0; i < counts.length; i++) { + if (counts[i] <= 0L) { + continue; + } + final var start = queryStart + (long) Math.floor(i * bucketSpan); + final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + bins.add(new TrackBin(start, Math.max(start + 1, end), values[i], counts[i], null)); + } + return bins; + } + + private static @NotNull List toBins(final @NotNull List projectedFeatures) { + return projectedFeatures.stream() + .map(f -> new TrackBin(f.start(), f.end(), f.value(), 1L, f.label())) + .toList(); + } + + private enum TrackType { + BED, + VCF, + BIGWIG, + BAM, + UNSUPPORTED; + + private static @NotNull TrackType fromPath(final @NotNull Path path) { + final var lowered = path.getFileName().toString().toLowerCase(Locale.ROOT); + if (lowered.endsWith(".bed") || lowered.endsWith(".bed.gz")) { + return BED; + } + if (lowered.endsWith(".vcf") || lowered.endsWith(".vcf.gz")) { + return VCF; + } + if (lowered.endsWith(".bw") || lowered.endsWith(".bigwig")) { + return BIGWIG; + } + if (lowered.endsWith(".bam")) { + return BAM; + } + return UNSUPPORTED; + } + } + + private enum BamRenderMode { + COVERAGE, + READ_DENSITY + } + + private enum BigWigAggregationMode { + MAX, + MEAN, + SUM + } + + private interface TrackDataSource extends AutoCloseable { + long featureCountHint(); + + @NotNull + List projectFeatures(@NotNull Map> sourceToAssemblySegments, + long queryStart, + long queryEnd); + + @Override + default void close() throws Exception { + // no-op + } + } + + private record InMemoryTrackDataSource(@NotNull Map> featuresBySource, + long featureCount) implements TrackDataSource { + static @NotNull InMemoryTrackDataSource fromBed(final @NotNull Path filePath) { + final var features = new HashMap>(); + long total = 0L; + try (final BufferedReader reader = openMaybeGzipReader(filePath)) { + String line; + long lineNo = 0L; + while ((line = reader.readLine()) != null) { + lineNo++; + if (line.isBlank() || line.startsWith("#") || line.startsWith("track ") || line.startsWith("browser ")) { + continue; + } + final var fields = line.split("\t"); + if (fields.length < 3) { + continue; + } + final var sourceName = fields[0]; + final long start = parseLongOrThrow(fields[1], "BED start", lineNo); + final long end = parseLongOrThrow(fields[2], "BED end", lineNo); + if (end <= start) { + continue; + } + final var label = (fields.length >= 4 && !fields[3].isBlank()) ? fields[3] : null; + final var value = parseOptionalDouble(fields.length >= 5 ? fields[4] : null, 1.0d); + features.computeIfAbsent(sourceName, ignored -> new ArrayList<>()) + .add(new FeatureRange(start, end, Math.max(0.0d, value), label)); + total++; + } + } catch (final IOException e) { + throw new RuntimeException("Failed to parse BED track " + filePath, e); + } + features.values().forEach(list -> list.sort(Comparator.comparingLong(FeatureRange::start))); + return new InMemoryTrackDataSource(features, total); + } + + static @NotNull InMemoryTrackDataSource fromVcf(final @NotNull Path filePath) { + final var features = new HashMap>(); + long total = 0L; + try (final BufferedReader reader = openMaybeGzipReader(filePath)) { + String line; + long lineNo = 0L; + while ((line = reader.readLine()) != null) { + lineNo++; + if (line.isBlank() || line.startsWith("#")) { + continue; + } + final var fields = line.split("\t"); + if (fields.length < 5) { + continue; + } + final var sourceName = fields[0]; + final long pos1Based = parseLongOrThrow(fields[1], "VCF POS", lineNo); + final long start = Math.max(0L, pos1Based - 1L); + final var ref = fields[3]; + final long end = start + Math.max(1, ref.length()); + final var id = (fields[2] != null && !fields[2].isBlank() && !".".equals(fields[2])) ? fields[2] : null; + final var alt = fields[4]; + final var label = (id != null) ? id : (ref + ">" + alt); + features.computeIfAbsent(sourceName, ignored -> new ArrayList<>()) + .add(new FeatureRange(start, end, 1.0d, label)); + total++; + } + } catch (final IOException e) { + throw new RuntimeException("Failed to parse VCF track " + filePath, e); + } + features.values().forEach(list -> list.sort(Comparator.comparingLong(FeatureRange::start))); + return new InMemoryTrackDataSource(features, total); + } + + @Override + public long featureCountHint() { + return this.featureCount; + } + + @Override + public @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd) { + final var projected = new ArrayList(); + for (final var entry : this.featuresBySource.entrySet()) { + final var sourceName = entry.getKey(); + final var sourceFeatures = entry.getValue(); + if (sourceFeatures.isEmpty()) { + continue; + } + final var assemblySegments = sourceToAssemblySegments.get(sourceName); + if (assemblySegments == null || assemblySegments.isEmpty()) { + continue; + } + for (final var segment : assemblySegments) { + int index = lowerBoundByStart(sourceFeatures, segment.sourceStart()); + if (index > 0) { + index--; + } + for (int i = index; i < sourceFeatures.size(); i++) { + final var feature = sourceFeatures.get(i); + if (feature.start() >= segment.sourceEnd()) { + break; + } + if (feature.end() <= segment.sourceStart()) { + continue; + } + projectSourceIntervalOnSegment( + segment, + feature.start(), + feature.end(), + feature.value(), + feature.label(), + queryStart, + queryEnd + ).ifPresent(projected::add); + if (projected.size() > MAX_FEATURES_PER_QUERY) { + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + } + } + } + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + + private static int lowerBoundByStart(final @NotNull List features, final long targetStart) { + int lo = 0; + int hi = features.size(); + while (lo < hi) { + final int mid = (lo + hi) >>> 1; + if (features.get(mid).start() < targetStart) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo; + } + } + + private static final class BigWigTrackDataSource implements TrackDataSource { + private final @NotNull Path path; + private final @NotNull BBFileReader reader; + private final @NotNull Set sourceNames; + + private BigWigTrackDataSource(final @NotNull Path path) { + this.path = path; + try { + this.reader = new BBFileReader(path.toString()); + if (!this.reader.isBigWigFile()) { + closeQuietly(); + throw new IllegalArgumentException("File " + path + " is not a BigWig"); + } + this.sourceNames = new HashSet<>(this.reader.getChromosomeNames()); + } catch (final IOException e) { + throw new RuntimeException("Failed to open BigWig " + path, e); + } + } + + @Override + public long featureCountHint() { + return -1L; + } + + @Override + public synchronized @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd) { + final var projected = new ArrayList(); + for (final var entry : sourceToAssemblySegments.entrySet()) { + final var sourceName = entry.getKey(); + if (!this.sourceNames.contains(sourceName)) { + continue; + } + for (final var segment : entry.getValue()) { + final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + final var queryStartClamped = clampToInt(sourceInterval.start()); + final var queryEndClamped = clampToInt(sourceInterval.end()); + if (queryEndClamped <= queryStartClamped) { + continue; + } + final BigWigIterator iterator = this.reader.getBigWigIterator( + sourceName, + queryStartClamped, + sourceName, + queryEndClamped, + false + ); + if (iterator == null) { + continue; + } + while (iterator.hasNext()) { + final WigItem item = iterator.next(); + final var itemStart = item.getStartBase(); + final var itemEnd = item.getEndBase(); + projectSourceIntervalOnSegment( + segment, + itemStart, + itemEnd, + Math.abs(item.getWigValue()), + null, + queryStart, + queryEnd + ).ifPresent(projected::add); + if (projected.size() > MAX_FEATURES_PER_QUERY) { + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + } + } + } + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + + public synchronized @NotNull List queryBins(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd, + final int widthPx, + final @NotNull BigWigAggregationMode mode) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] maxValues = new double[bucketCount]; + final double[] weightedSums = new double[bucketCount]; + final double[] overlapSums = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + for (final var entry : sourceToAssemblySegments.entrySet()) { + final var sourceName = entry.getKey(); + if (!this.sourceNames.contains(sourceName)) { + continue; + } + for (final var segment : entry.getValue()) { + final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + final var queryStartClamped = clampToInt(sourceInterval.start()); + final var queryEndClamped = clampToInt(sourceInterval.end()); + if (queryEndClamped <= queryStartClamped) { + continue; + } + final BigWigIterator iterator = this.reader.getBigWigIterator( + sourceName, + queryStartClamped, + sourceName, + queryEndClamped, + false + ); + if (iterator == null) { + continue; + } + while (iterator.hasNext()) { + final WigItem item = iterator.next(); + final var projectedFeature = projectSourceIntervalOnSegment( + segment, + item.getStartBase(), + item.getEndBase(), + Math.abs(item.getWigValue()), + null, + queryStart, + queryEnd + ); + if (projectedFeature.isEmpty()) { + continue; + } + final var feature = projectedFeature.get(); + accumulateBigWigValue( + feature.start(), + feature.end(), + feature.value(), + queryStart, + queryEnd, + bucketSpan, + maxValues, + weightedSums, + overlapSums, + counts + ); + } + } + } + return finalizeBigWigBins( + queryStart, + queryEnd, + bucketSpan, + maxValues, + weightedSums, + overlapSums, + counts, + mode + ); + } + + @Override + public void close() throws Exception { + if (this.reader.getBBFis() != null) { + this.reader.getBBFis().close(); + } + } + + private void closeQuietly() { + try { + close(); + } catch (final Exception ignored) { + } + } + + private static int clampToInt(final long value) { + if (value <= 0L) { + return 0; + } + if (value >= Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return (int) value; + } + } + + private static final class BamTrackDataSource implements TrackDataSource { + private final @NotNull Path path; + private final @NotNull SamReader reader; + private final @NotNull Set sequenceNames; + + private BamTrackDataSource(final @NotNull Path path) { + this.path = path; + final SamReaderFactory factory = SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT); + try { + this.reader = factory.open(path.toFile()); + if (!this.reader.hasIndex()) { + this.reader.close(); + throw new IllegalArgumentException( + "Failed to open BAM " + path.getFileName() + ": BAM index (.bai) is required and must be readable." + ); + } + } catch (final RuntimeException ex) { + throw new RuntimeException("Failed to open BAM " + path + ". Ensure BAM index (.bai) is present and readable.", ex); + } catch (final IOException ex) { + throw new RuntimeException("Failed to open BAM " + path + ". Ensure BAM index (.bai) is present and readable.", ex); + } + final SAMSequenceDictionary dictionary = this.reader.getFileHeader().getSequenceDictionary(); + final var names = new HashSet(); + dictionary.getSequences().forEach(seq -> names.add(seq.getSequenceName())); + this.sequenceNames = names; + } + + @Override + public long featureCountHint() { + return -1L; + } + + @Override + public synchronized @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd) { + final var projected = new ArrayList(); + for (final var entry : sourceToAssemblySegments.entrySet()) { + final var sourceName = entry.getKey(); + if (!this.sequenceNames.contains(sourceName)) { + continue; + } + for (final var segment : entry.getValue()) { + final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + final int startInclusive1 = Math.max(1, clampToInt(sourceInterval.start()) + 1); + final int endInclusive1 = Math.max(startInclusive1, clampToInt(sourceInterval.end())); + try (final CloseableIterator iterator = this.reader.query(sourceName, startInclusive1, endInclusive1, false)) { + while (iterator.hasNext()) { + final SAMRecord record = iterator.next(); + if (record.getReadUnmappedFlag()) { + continue; + } + final long recordStart = Math.max(0L, record.getAlignmentStart() - 1L); + final long recordEnd = Math.max(recordStart + 1L, record.getAlignmentEnd()); + projectSourceIntervalOnSegment( + segment, + recordStart, + recordEnd, + 1.0d, + null, + queryStart, + queryEnd + ).ifPresent(projected::add); + if (projected.size() > MAX_FEATURES_PER_QUERY) { + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + } + } + } + } + projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + return projected; + } + + public synchronized @NotNull List queryBins(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd, + final int widthPx, + final @NotNull BamRenderMode mode) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEnd - queryStart); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final double[] values = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + for (final var entry : sourceToAssemblySegments.entrySet()) { + final var sourceName = entry.getKey(); + if (!this.sequenceNames.contains(sourceName)) { + continue; + } + for (final var segment : entry.getValue()) { + final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + final int startInclusive1 = Math.max(1, clampToInt(sourceInterval.start()) + 1); + final int endInclusive1 = Math.max(startInclusive1, clampToInt(sourceInterval.end())); + try (final CloseableIterator iterator = this.reader.query(sourceName, startInclusive1, endInclusive1, false)) { + while (iterator.hasNext()) { + final SAMRecord record = iterator.next(); + if (record.getReadUnmappedFlag()) { + continue; + } + final var projectedFeature = projectSourceIntervalOnSegment( + segment, + Math.max(0L, record.getAlignmentStart() - 1L), + Math.max(Math.max(0L, record.getAlignmentStart() - 1L) + 1L, record.getAlignmentEnd()), + 1.0d, + null, + queryStart, + queryEnd + ); + if (projectedFeature.isEmpty()) { + continue; + } + final var feature = projectedFeature.get(); + if (mode == BamRenderMode.READ_DENSITY) { + accumulateReadDensityValue( + feature.start(), + feature.end(), + queryStart, + bucketSpan, + values, + counts + ); + } else { + accumulateCoverageValue( + feature.start(), + feature.end(), + queryStart, + queryEnd, + bucketSpan, + values, + counts + ); + } + } + } + } + } + return finalizeBins(queryStart, queryEnd, bucketSpan, values, counts); + } + + @Override + public void close() throws Exception { + this.reader.close(); + } + + private static int clampToInt(final long value) { + if (value <= 0L) { + return 0; + } + if (value >= Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return (int) value; + } + } + + private record SourceInterval(long start, long end) { + } + + private record AssemblySegment(long sourceStart, long sourceEnd, long assemblyStart, long assemblyEnd, boolean reversed) { + } + + private record FeatureRange(long start, long end, double value, String label) { + } + + private record ProjectedFeature(long start, long end, double value, String label) { + } + + @Getter + @RequiredArgsConstructor + public static final class QueryResult { + private final long startBp; + private final long endBp; + private final int widthPx; + private final @NotNull List tracks; + } + + @Getter + @RequiredArgsConstructor + public static final class TrackSummary { + private final @NotNull String trackId; + private final @NotNull String name; + private final @NotNull String type; + private final @NotNull String sourceFile; + private final @NotNull String color; + private final boolean visible; + private final long featureCount; + private final @NotNull String renderMode; + private final @NotNull String aggregationMode; + } + + @Getter + @RequiredArgsConstructor + public static final class TrackRender { + private final @NotNull String trackId; + private final @NotNull String name; + private final @NotNull String type; + private final @NotNull String color; + private final @NotNull List bins; + private final double maxValue; + private final String error; + } + + @Getter + @RequiredArgsConstructor + public static final class TrackBin { + private final long startBp; + private final long endBp; + private final double value; + private final long count; + private final String label; + } + + private record TrackState(@NotNull String trackId, + @NotNull String name, + @NotNull TrackType type, + @NotNull String sourceFile, + @NotNull String color, + boolean visible, + @NotNull TrackDataSource dataSource, + @NotNull BamRenderMode bamRenderMode, + @NotNull BigWigAggregationMode bigWigAggregationMode) { + private TrackSummary toSummary() { + return new TrackSummary( + trackId, + name, + type.name(), + sourceFile, + color, + visible, + dataSource.featureCountHint(), + bamRenderMode.name(), + bigWigAggregationMode.name() + ); + } + + private TrackRender toErrorRender(final @NotNull String message) { + return new TrackRender( + trackId, + name, + type.name(), + color, + List.of(), + 0.0d, + message + ); + } + + private TrackState withUpdated(final boolean newVisible, + final @NotNull String newColor, + final @NotNull String newName, + final @NotNull BamRenderMode newBamRenderMode, + final @NotNull BigWigAggregationMode newBigWigAggregationMode) { + return new TrackState( + trackId, + newName, + type, + sourceFile, + newColor, + newVisible, + dataSource, + newBamRenderMode, + newBigWigAggregationMode + ); + } + + private TrackRender query(final @NotNull Map> sourceToAssemblySegments, + final long queryStart, + final long queryEnd, + final int widthPx) { + final List bins; + if (type == TrackType.BAM && dataSource instanceof BamTrackDataSource bamDataSource) { + bins = bamDataSource.queryBins(sourceToAssemblySegments, queryStart, queryEnd, widthPx, bamRenderMode); + } else if (type == TrackType.BIGWIG && dataSource instanceof BigWigTrackDataSource bigWigDataSource) { + bins = bigWigDataSource.queryBins(sourceToAssemblySegments, queryStart, queryEnd, widthPx, bigWigAggregationMode); + } else { + final var projectedFeatures = dataSource.projectFeatures(sourceToAssemblySegments, queryStart, queryEnd); + final var maxFeatureCount = Math.max(widthPx * 8, 8192); + if (projectedFeatures.size() > maxFeatureCount) { + bins = aggregateFeatures(projectedFeatures, queryStart, queryEnd, widthPx); + } else { + bins = toBins(projectedFeatures); + } + } + final var maxValue = bins.stream().mapToDouble(TrackBin::getValue).max().orElse(0.0d); + return new TrackRender(trackId, name, type.name(), color, bins, maxValue, null); + } + } +} diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java index ded572c..e4a9b79 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java @@ -30,6 +30,7 @@ import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_library.visualization.SimpleVisualizationOptions; +import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; import java.nio.file.Path; @@ -51,4 +52,10 @@ public static class PathWrapper implements Shareable { public static class SimpleVisualizationOptionsWrapper implements Shareable { private final @NotNull SimpleVisualizationOptions simpleVisualizationOptions; } + + @Getter + @RequiredArgsConstructor + public static class Track1DManagerWrapper implements Shareable { + private final @NotNull Track1DManager track1DManager; + } } diff --git a/version.txt b/version.txt index 91af1b0..6aa072e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.55-d49cccd-webui_1a02368 \ No newline at end of file +1.0.83-1ce0372-webui_1a02368 \ No newline at end of file From aa7deb2c67d4fcfd1f37091d2d6a2c9383e5a53b Mon Sep 17 00:00:00 2001 From: Alexander Serdyukov Date: Sat, 21 Mar 2026 02:05:58 +0400 Subject: [PATCH 2/5] Intermediate fixes for 1D tracks renderer --- .../handlers/tracks/TrackHandlersHolder.java | 7 +- .../hict_server/tracks/Track1DManager.java | 558 +++++++++++++----- 2 files changed, 412 insertions(+), 153 deletions(-) diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java index 2b06eef..51678e9 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java @@ -128,9 +128,10 @@ public void addHandlersToRouter(final @NotNull Router router) { router.post("/tracks/query_1d").blockingHandler(ctx -> { final var request = ctx.body().asJsonObject(); - final var startBp = request.getLong("startBp", 0L); - final var endBp = request.getLong("endBp", startBp + 1L); + final var startPx = request.getLong("startPx", 0L); + final var endPx = request.getLong("endPx", startPx + 1L); final var widthPx = request.getInteger("widthPx", 512); + final var bpResolution = request.getLong("bpResolution", 1L); final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); final var manager = getTrackManager(ctx); @@ -141,7 +142,7 @@ public void addHandlersToRouter(final @NotNull Router router) { if (chunkedFile == null) { return; } - final var result = manager.queryVisibleTracks(chunkedFile, startBp, endBp, widthPx); + final var result = manager.queryVisibleTracks(chunkedFile, startPx, endPx, widthPx, bpResolution); ctx.response() .putHeader("content-type", "application/json") .end(Json.encode(result)); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java index 20b5745..7adda3e 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java @@ -38,9 +38,9 @@ import org.broad.igv.bbfile.WigItem; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import ru.itmo.ctlab.hict.hict_library.assembly.FASTAProcessor; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_library.domain.ContigDirection; +import ru.itmo.ctlab.hict.hict_library.domain.ContigHideType; import ru.itmo.ctlab.hict.hict_library.trees.ContigTree; import java.io.BufferedReader; @@ -54,7 +54,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.IntStream; import java.util.zip.GZIPInputStream; @Slf4j @@ -199,14 +198,21 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al } public @NotNull QueryResult queryVisibleTracks(final @NotNull ChunkedFile chunkedFile, - final long startBp, - final long endBp, - final int widthPx) { - final var queryStart = Math.max(0L, Math.min(startBp, endBp)); - final var queryEnd = Math.max(queryStart + 1L, Math.max(startBp, endBp)); + final long startPx, + final long endPx, + final int widthPx, + final long bpResolution) { final var safeWidth = Math.max(1, widthPx); + final var segmentsBuildResult = + buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource, bpResolution); + final var totalVisiblePixels = segmentsBuildResult.totalVisiblePixels(); + if (totalVisiblePixels <= 0L) { + return new QueryResult(0L, 1L, 0L, 1L, safeWidth, bpResolution, List.of()); + } + final var queryStartPx = Math.max(0L, Math.min(Math.min(startPx, endPx), totalVisiblePixels - 1L)); + final var queryEndPx = Math.max(queryStartPx + 1L, Math.min(Math.max(startPx, endPx), totalVisiblePixels)); final Map> sourceToAssemblySegments = - buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource); + segmentsBuildResult.sourceToAssemblySegments(); final List trackRenders = new ArrayList<>(); try { this.lock.readLock().lock(); @@ -214,7 +220,7 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al .filter(track -> track.visible) .forEach(track -> { try { - trackRenders.add(track.query(sourceToAssemblySegments, queryStart, queryEnd, safeWidth)); + trackRenders.add(track.query(sourceToAssemblySegments, queryStartPx, queryEndPx, safeWidth, bpResolution)); } catch (final RuntimeException ex) { final var message = ex.getMessage() != null ? ex.getMessage() : ex.getClass().getSimpleName(); log.error("Failed to query 1D track {} ({})", track.name(), track.trackId(), ex); @@ -224,14 +230,27 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al } finally { this.lock.readLock().unlock(); } - return new QueryResult(queryStart, queryEnd, safeWidth, trackRenders); + final var startBp = mapVisiblePxToAssemblyBp(queryStartPx, segmentsBuildResult.orderedSegments(), bpResolution); + final var endBp = mapVisiblePxToAssemblyBp( + Math.max(queryStartPx, queryEndPx - 1L), + segmentsBuildResult.orderedSegments(), + bpResolution + ) + bpResolution; + return new QueryResult(startBp, endBp, queryStartPx, queryEndPx, safeWidth, bpResolution, trackRenders); } - private @NotNull Map> buildSourceToAssemblySegments(final @NotNull ChunkedFile chunkedFile, - final @NotNull Map linkedFastaAliasesBySource) { + private @NotNull SegmentBuildResult buildSourceToAssemblySegments(final @NotNull ChunkedFile chunkedFile, + final @NotNull Map linkedFastaAliasesBySource, + final long bpResolution) { + final var resolutionOrder = chunkedFile.getResolutionToIndex().get(bpResolution); + if (resolutionOrder == null) { + throw new IllegalArgumentException("Unsupported resolution for 1D track query: " + bpResolution); + } final var sourceToAssemblySegments = new HashMap>(); + final var orderedSegments = new ArrayList(); final var contigs = chunkedFile.getAssemblyInfo().contigs(); long assemblyCursor = 0L; + long visiblePxCursor = 0L; for (int contigIndex = 0; contigIndex < contigs.size(); ++contigIndex) { final ContigTree.ContigTuple tuple = contigs.get(contigIndex); final var descriptor = tuple.descriptor(); @@ -240,17 +259,30 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al final var sourceEnd = sourceStart + descriptor.getLengthBp(); final var assemblyStart = assemblyCursor; final var assemblyEnd = assemblyCursor + descriptor.getLengthBp(); - sourceToAssemblySegments.computeIfAbsent(sourceName, key -> new ArrayList<>()) - .add(new AssemblySegment(sourceStart, sourceEnd, assemblyStart, assemblyEnd, tuple.direction() == ContigDirection.REVERSED)); - final var aliasName = linkedFastaAliasesBySource.get(sourceName); - if (aliasName != null && !aliasName.equals(sourceName)) { - sourceToAssemblySegments.computeIfAbsent(aliasName, key -> new ArrayList<>()) - .add(new AssemblySegment(sourceStart, sourceEnd, assemblyStart, assemblyEnd, tuple.direction() == ContigDirection.REVERSED)); + final var visibleAtResolution = descriptor.getPresenceAtResolution().get(resolutionOrder) == ContigHideType.SHOWN; + final var lengthPxAtResolution = descriptor.getLengthBinsAtResolution()[resolutionOrder]; + if (visibleAtResolution && lengthPxAtResolution > 0L) { + final var segment = new AssemblySegment( + sourceStart, + sourceEnd, + assemblyStart, + assemblyEnd, + tuple.direction() == ContigDirection.REVERSED, + visiblePxCursor, + visiblePxCursor + lengthPxAtResolution + ); + orderedSegments.add(segment); + sourceToAssemblySegments.computeIfAbsent(sourceName, key -> new ArrayList<>()).add(segment); + final var aliasName = linkedFastaAliasesBySource.get(sourceName); + if (aliasName != null && !aliasName.equals(sourceName)) { + sourceToAssemblySegments.computeIfAbsent(aliasName, key -> new ArrayList<>()).add(segment); + } + visiblePxCursor += lengthPxAtResolution; } assemblyCursor = assemblyEnd; } sourceToAssemblySegments.values().forEach(list -> list.sort(Comparator.comparingLong(AssemblySegment::sourceStart))); - return sourceToAssemblySegments; + return new SegmentBuildResult(sourceToAssemblySegments, orderedSegments, visiblePxCursor); } private boolean isSupportedTrackPath(final @NotNull String path) { @@ -349,13 +381,108 @@ private static double parseOptionalDouble(final String value, final double defau return new BufferedReader(new InputStreamReader(dataStream, StandardCharsets.UTF_8)); } + private static long localPxToAssemblyBp(final @NotNull AssemblySegment segment, + final long localPx, + final long bpResolution) { + final var segmentLengthBp = segment.assemblyEnd() - segment.assemblyStart(); + if (segmentLengthBp <= 0L) { + return segment.assemblyStart(); + } + final var segmentLengthPx = Math.max(1L, segment.visiblePxEnd() - segment.visiblePxStart()); + final var clampedLocalPx = Math.max(0L, Math.min(localPx, segmentLengthPx - 1L)); + final long localBp; + if (!segment.reversed()) { + localBp = clampedLocalPx * bpResolution; + } else { + final var firstBinLengthBp = segmentLengthBp % bpResolution; + if (clampedLocalPx <= 0L) { + localBp = 0L; + } else { + localBp = firstBinLengthBp + (clampedLocalPx - 1L) * bpResolution; + } + } + return Math.max(segment.assemblyStart(), Math.min(segment.assemblyEnd() - 1L, segment.assemblyStart() + localBp)); + } + + private static long assemblyBpToLocalPx(final @NotNull AssemblySegment segment, + final long assemblyBp, + final long bpResolution) { + final var segmentLengthBp = segment.assemblyEnd() - segment.assemblyStart(); + if (segmentLengthBp <= 0L) { + return 0L; + } + final var segmentLengthPx = Math.max(1L, segment.visiblePxEnd() - segment.visiblePxStart()); + final var clampedBp = Math.max(segment.assemblyStart(), Math.min(assemblyBp, segment.assemblyEnd() - 1L)); + final var inContigOffsetBp = clampedBp - segment.assemblyStart(); + final long localPx; + if (!segment.reversed()) { + localPx = inContigOffsetBp / bpResolution; + } else { + final var firstBinLengthBp = segmentLengthBp % bpResolution; + if (inContigOffsetBp < firstBinLengthBp) { + localPx = 0L; + } else { + localPx = 1L + ((inContigOffsetBp - firstBinLengthBp) / bpResolution); + } + } + return Math.max(0L, Math.min(localPx, segmentLengthPx - 1L)); + } + + private static long mapVisiblePxToAssemblyBp(final long px, + final @NotNull List orderedSegments, + final long bpResolution) { + if (orderedSegments.isEmpty()) { + return 0L; + } + int lo = 0; + int hi = orderedSegments.size(); + while (lo < hi) { + final int mid = (lo + hi) >>> 1; + if (orderedSegments.get(mid).visiblePxEnd() <= px) { + lo = mid + 1; + } else { + hi = mid; + } + } + final int idx = Math.max(0, Math.min(lo, orderedSegments.size() - 1)); + AssemblySegment segment = orderedSegments.get(idx); + if (px < segment.visiblePxStart() && idx > 0) { + segment = orderedSegments.get(idx - 1); + } + final var localPx = Math.max(0L, px - segment.visiblePxStart()); + return localPxToAssemblyBp(segment, localPx, bpResolution); + } + + private static @NotNull Optional mapVisiblePxIntervalToSegmentSource(final @NotNull AssemblySegment segment, + final long queryStartPx, + final long queryEndPx, + final long bpResolution) { + final var overlapStartPx = Math.max(queryStartPx, segment.visiblePxStart()); + final var overlapEndPx = Math.min(queryEndPx, segment.visiblePxEnd()); + if (overlapEndPx <= overlapStartPx) { + return Optional.empty(); + } + final var localStartPx = overlapStartPx - segment.visiblePxStart(); + final var localEndPx = overlapEndPx - segment.visiblePxStart(); + final var assemblyStart = localPxToAssemblyBp(segment, localStartPx, bpResolution); + final var assemblyEnd = Math.min( + segment.assemblyEnd(), + localPxToAssemblyBp(segment, Math.max(localStartPx, localEndPx - 1L), bpResolution) + bpResolution + ); + if (assemblyEnd <= assemblyStart) { + return Optional.empty(); + } + return mapAssemblyIntervalToSegmentSource(segment, assemblyStart, assemblyEnd); + } + private static @NotNull Optional projectSourceIntervalOnSegment(final @NotNull AssemblySegment segment, final long sourceStart, final long sourceEnd, final double value, final String label, - final long queryStart, - final long queryEnd) { + final long queryStartPx, + final long queryEndPx, + final long bpResolution) { final var clippedSourceStart = Math.max(sourceStart, segment.sourceStart()); final var clippedSourceEnd = Math.min(sourceEnd, segment.sourceEnd()); if (clippedSourceEnd <= clippedSourceStart) { @@ -375,12 +502,32 @@ private static double parseOptionalDouble(final String value, final double defau assemblyEnd = segment.assemblyStart() + (segmentLength - localStart); } - final var clippedAssemblyStart = Math.max(queryStart, Math.min(assemblyStart, assemblyEnd)); - final var clippedAssemblyEnd = Math.min(queryEnd, Math.max(assemblyStart, assemblyEnd)); + final var clippedAssemblyStart = Math.max(segment.assemblyStart(), Math.min(assemblyStart, assemblyEnd)); + final var clippedAssemblyEnd = Math.min(segment.assemblyEnd(), Math.max(assemblyStart, assemblyEnd)); if (clippedAssemblyEnd <= clippedAssemblyStart) { return Optional.empty(); } - return Optional.of(new ProjectedFeature(clippedAssemblyStart, clippedAssemblyEnd, Math.max(0.0d, value), label)); + final var localStartPx = assemblyBpToLocalPx(segment, clippedAssemblyStart, bpResolution); + final var localEndPx = assemblyBpToLocalPx( + segment, + Math.max(clippedAssemblyStart, clippedAssemblyEnd - 1L), + bpResolution + ) + 1L; + final var featureStartPx = segment.visiblePxStart() + localStartPx; + final var featureEndPx = segment.visiblePxStart() + localEndPx; + final var clippedStartPx = Math.max(queryStartPx, featureStartPx); + final var clippedEndPx = Math.min(queryEndPx, featureEndPx); + if (clippedEndPx <= clippedStartPx) { + return Optional.empty(); + } + return Optional.of(new ProjectedFeature( + clippedAssemblyStart, + clippedAssemblyEnd, + clippedStartPx, + clippedEndPx, + Math.max(0.0d, value), + label + )); } private static @NotNull Optional mapAssemblyIntervalToSegmentSource(final @NotNull AssemblySegment segment, @@ -403,18 +550,18 @@ private static double parseOptionalDouble(final String value, final double defau } private static @NotNull List aggregateFeatures(final @NotNull List projectedFeatures, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] maxValues = new double[bucketCount]; final long[] counts = new long[bucketCount]; Arrays.fill(maxValues, 0.0d); for (final var feature : projectedFeatures) { - int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); - int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + int left = (int) Math.floor((feature.startPx() - queryStartPx) / bucketSpan); + int right = (int) Math.ceil((feature.endPx() - queryStartPx) / bucketSpan) - 1; left = Math.max(0, Math.min(left, bucketCount - 1)); right = Math.max(0, Math.min(right, bucketCount - 1)); for (int i = left; i <= right; i++) { @@ -427,20 +574,21 @@ private static double parseOptionalDouble(final String value, final double defau if (counts[i] <= 0) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - bins.add(new TrackBin(start, Math.max(start + 1, end), maxValues[i], counts[i], null)); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + bins.add(new TrackBin(startPx, safeEndPx, maxValues[i], counts[i], null, startPx, safeEndPx)); } return bins; } private static @NotNull List aggregateBigWigFeatures(final @NotNull List projectedFeatures, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx, final @NotNull BigWigAggregationMode mode) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] maxValues = new double[bucketCount]; final double[] weightedSums = new double[bucketCount]; @@ -448,14 +596,14 @@ private static double parseOptionalDouble(final String value, final double defau final long[] counts = new long[bucketCount]; Arrays.fill(maxValues, 0.0d); for (final var feature : projectedFeatures) { - int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); - int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + int left = (int) Math.floor((feature.startPx() - queryStartPx) / bucketSpan); + int right = (int) Math.ceil((feature.endPx() - queryStartPx) / bucketSpan) - 1; left = Math.max(0, Math.min(left, bucketCount - 1)); right = Math.max(0, Math.min(right, bucketCount - 1)); for (int i = left; i <= right; i++) { - final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); - final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - final var overlap = Math.min(feature.end(), bucketEnd) - Math.max(feature.start(), bucketStart); + final var bucketStart = queryStartPx + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(feature.endPx(), bucketEnd) - Math.max(feature.startPx(), bucketStart); if (overlap <= 0L) { continue; } @@ -470,38 +618,39 @@ private static double parseOptionalDouble(final String value, final double defau if (counts[i] <= 0) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - final var bucketWidth = Math.max(1.0d, end - start); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + final var bucketWidth = Math.max(1.0d, safeEndPx - startPx); final double value = switch (mode) { case MAX -> maxValues[i]; case MEAN -> overlapSums[i] > 0.0d ? weightedSums[i] / overlapSums[i] : 0.0d; case SUM -> weightedSums[i] / bucketWidth; }; - bins.add(new TrackBin(start, Math.max(start + 1, end), value, counts[i], null)); + bins.add(new TrackBin(startPx, safeEndPx, value, counts[i], null, startPx, safeEndPx)); } return bins; } private static @NotNull List aggregateCoverageFeatures(final @NotNull List projectedFeatures, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] coverage = new double[bucketCount]; final long[] counts = new long[bucketCount]; Arrays.fill(coverage, 0.0d); for (final var feature : projectedFeatures) { - int left = (int) Math.floor((feature.start() - queryStart) / bucketSpan); - int right = (int) Math.ceil((feature.end() - queryStart) / bucketSpan) - 1; + int left = (int) Math.floor((feature.startPx() - queryStartPx) / bucketSpan); + int right = (int) Math.ceil((feature.endPx() - queryStartPx) / bucketSpan) - 1; left = Math.max(0, Math.min(left, bucketCount - 1)); right = Math.max(0, Math.min(right, bucketCount - 1)); for (int i = left; i <= right; i++) { - final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); - final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - final var overlap = Math.min(feature.end(), bucketEnd) - Math.max(feature.start(), bucketStart); + final var bucketStart = queryStartPx + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var overlap = Math.min(feature.endPx(), bucketEnd) - Math.max(feature.startPx(), bucketStart); if (overlap <= 0L) { continue; } @@ -515,26 +664,27 @@ private static double parseOptionalDouble(final String value, final double defau if (counts[i] <= 0) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - bins.add(new TrackBin(start, Math.max(start + 1, end), coverage[i], counts[i], null)); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + bins.add(new TrackBin(startPx, safeEndPx, coverage[i], counts[i], null, startPx, safeEndPx)); } return bins; } private static @NotNull List aggregateReadDensityFeatures(final @NotNull List projectedFeatures, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] values = new double[bucketCount]; final long[] counts = new long[bucketCount]; Arrays.fill(values, 0.0d); for (final var feature : projectedFeatures) { - final var center = feature.start() + ((feature.end() - feature.start()) >>> 1); - int idx = (int) Math.floor((center - queryStart) / bucketSpan); + final var center = feature.startPx() + ((feature.endPx() - feature.startPx()) >>> 1); + int idx = (int) Math.floor((center - queryStartPx) / bucketSpan); idx = Math.max(0, Math.min(idx, bucketCount - 1)); values[idx] += 1.0d; counts[idx] += 1L; @@ -544,9 +694,10 @@ private static double parseOptionalDouble(final String value, final double defau if (counts[i] <= 0L) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - bins.add(new TrackBin(start, Math.max(start + 1, end), values[i], counts[i], null)); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + bins.add(new TrackBin(startPx, safeEndPx, values[i], counts[i], null, startPx, safeEndPx)); } return bins; } @@ -554,20 +705,20 @@ private static double parseOptionalDouble(final String value, final double defau private static void accumulateBigWigValue(final long featureStart, final long featureEnd, final double featureValue, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final double bucketSpan, final double[] maxValues, final double[] weightedSums, final double[] overlapSums, final long[] counts) { - int left = (int) Math.floor((featureStart - queryStart) / bucketSpan); - int right = (int) Math.ceil((featureEnd - queryStart) / bucketSpan) - 1; + int left = (int) Math.floor((featureStart - queryStartPx) / bucketSpan); + int right = (int) Math.ceil((featureEnd - queryStartPx) / bucketSpan) - 1; left = Math.max(0, Math.min(left, counts.length - 1)); right = Math.max(0, Math.min(right, counts.length - 1)); for (int i = left; i <= right; i++) { - final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); - final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var bucketStart = queryStartPx + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); final var overlap = Math.min(featureEnd, bucketEnd) - Math.max(featureStart, bucketStart); if (overlap <= 0L) { continue; @@ -581,18 +732,18 @@ private static void accumulateBigWigValue(final long featureStart, private static void accumulateCoverageValue(final long featureStart, final long featureEnd, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final double bucketSpan, final double[] coverage, final long[] counts) { - int left = (int) Math.floor((featureStart - queryStart) / bucketSpan); - int right = (int) Math.ceil((featureEnd - queryStart) / bucketSpan) - 1; + int left = (int) Math.floor((featureStart - queryStartPx) / bucketSpan); + int right = (int) Math.ceil((featureEnd - queryStartPx) / bucketSpan) - 1; left = Math.max(0, Math.min(left, counts.length - 1)); right = Math.max(0, Math.min(right, counts.length - 1)); for (int i = left; i <= right; i++) { - final var bucketStart = queryStart + (long) Math.floor(i * bucketSpan); - final var bucketEnd = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); + final var bucketStart = queryStartPx + (long) Math.floor(i * bucketSpan); + final var bucketEnd = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); final var overlap = Math.min(featureEnd, bucketEnd) - Math.max(featureStart, bucketStart); if (overlap <= 0L) { continue; @@ -604,19 +755,19 @@ private static void accumulateCoverageValue(final long featureStart, private static void accumulateReadDensityValue(final long featureStart, final long featureEnd, - final long queryStart, + final long queryStartPx, final double bucketSpan, final double[] values, final long[] counts) { final var center = featureStart + ((featureEnd - featureStart) >>> 1); - int idx = (int) Math.floor((center - queryStart) / bucketSpan); + int idx = (int) Math.floor((center - queryStartPx) / bucketSpan); idx = Math.max(0, Math.min(idx, counts.length - 1)); values[idx] += 1.0d; counts[idx] += 1L; } - private static @NotNull List finalizeBigWigBins(final long queryStart, - final long queryEnd, + private static @NotNull List finalizeBigWigBins(final long queryStartPx, + final long queryEndPx, final double bucketSpan, final double[] maxValues, final double[] weightedSums, @@ -628,21 +779,22 @@ private static void accumulateReadDensityValue(final long featureStart, if (counts[i] <= 0L) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - final var bucketWidth = Math.max(1.0d, end - start); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + final var bucketWidth = Math.max(1.0d, safeEndPx - startPx); final double value = switch (mode) { case MAX -> maxValues[i]; case MEAN -> overlapSums[i] > 0.0d ? weightedSums[i] / overlapSums[i] : 0.0d; case SUM -> weightedSums[i] / bucketWidth; }; - bins.add(new TrackBin(start, Math.max(start + 1, end), value, counts[i], null)); + bins.add(new TrackBin(startPx, safeEndPx, value, counts[i], null, startPx, safeEndPx)); } return bins; } - private static @NotNull List finalizeBins(final long queryStart, - final long queryEnd, + private static @NotNull List finalizeBins(final long queryStartPx, + final long queryEndPx, final double bucketSpan, final double[] values, final long[] counts) { @@ -651,16 +803,17 @@ private static void accumulateReadDensityValue(final long featureStart, if (counts[i] <= 0L) { continue; } - final var start = queryStart + (long) Math.floor(i * bucketSpan); - final var end = Math.min(queryEnd, queryStart + (long) Math.ceil((i + 1) * bucketSpan)); - bins.add(new TrackBin(start, Math.max(start + 1, end), values[i], counts[i], null)); + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + bins.add(new TrackBin(startPx, safeEndPx, values[i], counts[i], null, startPx, safeEndPx)); } return bins; } private static @NotNull List toBins(final @NotNull List projectedFeatures) { return projectedFeatures.stream() - .map(f -> new TrackBin(f.start(), f.end(), f.value(), 1L, f.label())) + .map(f -> new TrackBin(f.startBp(), f.endBp(), f.value(), 1L, f.label(), f.startPx(), f.endPx())) .toList(); } @@ -705,8 +858,9 @@ private interface TrackDataSource extends AutoCloseable { @NotNull List projectFeatures(@NotNull Map> sourceToAssemblySegments, - long queryStart, - long queryEnd); + long queryStartPx, + long queryEndPx, + long bpResolution); @Override default void close() throws Exception { @@ -791,8 +945,9 @@ public long featureCountHint() { @Override public @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd) { + final long queryStartPx, + final long queryEndPx, + final long bpResolution) { final var projected = new ArrayList(); for (final var entry : this.featuresBySource.entrySet()) { final var sourceName = entry.getKey(); @@ -805,16 +960,26 @@ public long featureCountHint() { continue; } for (final var segment : assemblySegments) { - int index = lowerBoundByStart(sourceFeatures, segment.sourceStart()); + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + int index = lowerBoundByStart(sourceFeatures, sourceInterval.start()); if (index > 0) { index--; } for (int i = index; i < sourceFeatures.size(); i++) { final var feature = sourceFeatures.get(i); - if (feature.start() >= segment.sourceEnd()) { + if (feature.start() >= sourceInterval.end()) { break; } - if (feature.end() <= segment.sourceStart()) { + if (feature.end() <= sourceInterval.start()) { continue; } projectSourceIntervalOnSegment( @@ -823,17 +988,18 @@ public long featureCountHint() { feature.end(), feature.value(), feature.label(), - queryStart, - queryEnd + queryStartPx, + queryEndPx, + bpResolution ).ifPresent(projected::add); if (projected.size() > MAX_FEATURES_PER_QUERY) { - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } } } } - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } @@ -878,8 +1044,9 @@ public long featureCountHint() { @Override public synchronized @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd) { + final long queryStartPx, + final long queryEndPx, + final long bpResolution) { final var projected = new ArrayList(); for (final var entry : sourceToAssemblySegments.entrySet()) { final var sourceName = entry.getKey(); @@ -887,7 +1054,12 @@ public long featureCountHint() { continue; } for (final var segment : entry.getValue()) { - final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); if (sourceIntervalOptional.isEmpty()) { continue; } @@ -917,27 +1089,29 @@ public long featureCountHint() { itemEnd, Math.abs(item.getWigValue()), null, - queryStart, - queryEnd + queryStartPx, + queryEndPx, + bpResolution ).ifPresent(projected::add); if (projected.size() > MAX_FEATURES_PER_QUERY) { - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } } } } - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } public synchronized @NotNull List queryBins(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx, + final long bpResolution, final @NotNull BigWigAggregationMode mode) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] maxValues = new double[bucketCount]; final double[] weightedSums = new double[bucketCount]; @@ -949,7 +1123,12 @@ public long featureCountHint() { continue; } for (final var segment : entry.getValue()) { - final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); if (sourceIntervalOptional.isEmpty()) { continue; } @@ -977,19 +1156,20 @@ public long featureCountHint() { item.getEndBase(), Math.abs(item.getWigValue()), null, - queryStart, - queryEnd + queryStartPx, + queryEndPx, + bpResolution ); if (projectedFeature.isEmpty()) { continue; } final var feature = projectedFeature.get(); accumulateBigWigValue( - feature.start(), - feature.end(), + feature.startPx(), + feature.endPx(), feature.value(), - queryStart, - queryEnd, + queryStartPx, + queryEndPx, bucketSpan, maxValues, weightedSums, @@ -1000,8 +1180,8 @@ public long featureCountHint() { } } return finalizeBigWigBins( - queryStart, - queryEnd, + queryStartPx, + queryEndPx, bucketSpan, maxValues, weightedSums, @@ -1070,8 +1250,9 @@ public long featureCountHint() { @Override public synchronized @NotNull List projectFeatures(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd) { + final long queryStartPx, + final long queryEndPx, + final long bpResolution) { final var projected = new ArrayList(); for (final var entry : sourceToAssemblySegments.entrySet()) { final var sourceName = entry.getKey(); @@ -1079,7 +1260,12 @@ public long featureCountHint() { continue; } for (final var segment : entry.getValue()) { - final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); if (sourceIntervalOptional.isEmpty()) { continue; } @@ -1100,28 +1286,30 @@ public long featureCountHint() { recordEnd, 1.0d, null, - queryStart, - queryEnd + queryStartPx, + queryEndPx, + bpResolution ).ifPresent(projected::add); if (projected.size() > MAX_FEATURES_PER_QUERY) { - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } } } } } - projected.sort(Comparator.comparingLong(ProjectedFeature::start)); + projected.sort(Comparator.comparingLong(ProjectedFeature::startPx)); return projected; } public synchronized @NotNull List queryBins(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd, + final long queryStartPx, + final long queryEndPx, final int widthPx, + final long bpResolution, final @NotNull BamRenderMode mode) { final var bucketCount = Math.max(1, widthPx); - final var span = Math.max(1L, queryEnd - queryStart); + final var span = Math.max(1L, queryEndPx - queryStartPx); final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); final double[] values = new double[bucketCount]; final long[] counts = new long[bucketCount]; @@ -1131,7 +1319,12 @@ public long featureCountHint() { continue; } for (final var segment : entry.getValue()) { - final var sourceIntervalOptional = mapAssemblyIntervalToSegmentSource(segment, queryStart, queryEnd); + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); if (sourceIntervalOptional.isEmpty()) { continue; } @@ -1150,8 +1343,9 @@ public long featureCountHint() { Math.max(Math.max(0L, record.getAlignmentStart() - 1L) + 1L, record.getAlignmentEnd()), 1.0d, null, - queryStart, - queryEnd + queryStartPx, + queryEndPx, + bpResolution ); if (projectedFeature.isEmpty()) { continue; @@ -1159,19 +1353,19 @@ public long featureCountHint() { final var feature = projectedFeature.get(); if (mode == BamRenderMode.READ_DENSITY) { accumulateReadDensityValue( - feature.start(), - feature.end(), - queryStart, + feature.startPx(), + feature.endPx(), + queryStartPx, bucketSpan, values, counts ); } else { accumulateCoverageValue( - feature.start(), - feature.end(), - queryStart, - queryEnd, + feature.startPx(), + feature.endPx(), + queryStartPx, + queryEndPx, bucketSpan, values, counts @@ -1181,7 +1375,7 @@ public long featureCountHint() { } } } - return finalizeBins(queryStart, queryEnd, bucketSpan, values, counts); + return finalizeBins(queryStartPx, queryEndPx, bucketSpan, values, counts); } @Override @@ -1203,13 +1397,29 @@ private static int clampToInt(final long value) { private record SourceInterval(long start, long end) { } - private record AssemblySegment(long sourceStart, long sourceEnd, long assemblyStart, long assemblyEnd, boolean reversed) { + private record SegmentBuildResult(@NotNull Map> sourceToAssemblySegments, + @NotNull List orderedSegments, + long totalVisiblePixels) { + } + + private record AssemblySegment(long sourceStart, + long sourceEnd, + long assemblyStart, + long assemblyEnd, + boolean reversed, + long visiblePxStart, + long visiblePxEnd) { } private record FeatureRange(long start, long end, double value, String label) { } - private record ProjectedFeature(long start, long end, double value, String label) { + private record ProjectedFeature(long startBp, + long endBp, + long startPx, + long endPx, + double value, + String label) { } @Getter @@ -1217,7 +1427,10 @@ private record ProjectedFeature(long start, long end, double value, String label public static final class QueryResult { private final long startBp; private final long endBp; + private final long startPx; + private final long endPx; private final int widthPx; + private final long bpResolution; private final @NotNull List tracks; } @@ -1248,13 +1461,38 @@ public static final class TrackRender { } @Getter - @RequiredArgsConstructor public static final class TrackBin { private final long startBp; private final long endBp; private final double value; private final long count; private final String label; + private final Long startPx; + private final Long endPx; + + public TrackBin(final long startBp, + final long endBp, + final double value, + final long count, + final String label) { + this(startBp, endBp, value, count, label, null, null); + } + + public TrackBin(final long startBp, + final long endBp, + final double value, + final long count, + final String label, + final Long startPx, + final Long endPx) { + this.startBp = startBp; + this.endBp = endBp; + this.value = value; + this.count = count; + this.label = label; + this.startPx = startPx; + this.endPx = endPx; + } } private record TrackState(@NotNull String trackId, @@ -1311,19 +1549,39 @@ private TrackState withUpdated(final boolean newVisible, } private TrackRender query(final @NotNull Map> sourceToAssemblySegments, - final long queryStart, - final long queryEnd, - final int widthPx) { + final long queryStartPx, + final long queryEndPx, + final int widthPx, + final long bpResolution) { final List bins; if (type == TrackType.BAM && dataSource instanceof BamTrackDataSource bamDataSource) { - bins = bamDataSource.queryBins(sourceToAssemblySegments, queryStart, queryEnd, widthPx, bamRenderMode); + bins = bamDataSource.queryBins( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution, + bamRenderMode + ); } else if (type == TrackType.BIGWIG && dataSource instanceof BigWigTrackDataSource bigWigDataSource) { - bins = bigWigDataSource.queryBins(sourceToAssemblySegments, queryStart, queryEnd, widthPx, bigWigAggregationMode); + bins = bigWigDataSource.queryBins( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution, + bigWigAggregationMode + ); } else { - final var projectedFeatures = dataSource.projectFeatures(sourceToAssemblySegments, queryStart, queryEnd); + final var projectedFeatures = dataSource.projectFeatures( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + bpResolution + ); final var maxFeatureCount = Math.max(widthPx * 8, 8192); if (projectedFeatures.size() > maxFeatureCount) { - bins = aggregateFeatures(projectedFeatures, queryStart, queryEnd, widthPx); + bins = aggregateFeatures(projectedFeatures, queryStartPx, queryEndPx, widthPx); } else { bins = toBins(projectedFeatures); } From ad8acb7224e580fe6da0fc2e8527abf7b41b192e Mon Sep 17 00:00:00 2001 From: Alexander Serdyukov Date: Sat, 21 Mar 2026 03:16:49 +0400 Subject: [PATCH 3/5] 1D tracks with precomputing but not smooth --- build.gradle.kts | 13 +- .../ctlab/hict/hict_server/MainVerticle.java | 6 +- .../handlers/fileop/FileOpHandlersHolder.java | 6 +- .../handlers/tracks/TrackHandlersHolder.java | 130 ++- .../hict_server/tracks/Track1DManager.java | 826 +++++++++++++++++- version.txt | 2 +- 6 files changed, 929 insertions(+), 54 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3796363..6658928 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -71,7 +71,7 @@ val webUIRepositoryDirectory = if (localWebUIRepositoryDirectory.asFile.exists()) localWebUIRepositoryDirectory else remoteWebUIRepositoryDirectory val webUIRepositoryAddress = "https://github.com/ctlab/HiCT_WebUI.git" val webUITargetDirectory = layout.projectDirectory.dir("src/main/resources/webui") -val webUIBranch = "dev-0.1.5" +val webUIBranch = "migrate-converters-update-ui-1dtracks" version = readVersion() @@ -257,13 +257,10 @@ tasks.register("buildWebUI") { doLast { try { if (localWebUIRepositoryDirectory.asFile.exists()) { - println("Using local HiCT_WebUI checkout at ${localWebUIRepositoryDirectory.asFile.absolutePath}") - project.exec { - commandLine("git", "checkout", webUIBranch) - workingDir = localWebUIRepositoryDirectory.asFile - standardOutput = System.out - isIgnoreExitValue = true - } + println( + "Using local HiCT_WebUI checkout at ${localWebUIRepositoryDirectory.asFile.absolutePath} " + + "(branch/working tree will not be modified by Gradle)" + ) project.exec { commandLine("npm", "install") workingDir = localWebUIRepositoryDirectory.asFile diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java index 6190262..037b708 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java @@ -93,7 +93,7 @@ public void start(final Promise startPromise) throws Exception { final ConfigStoreOptions jsonEnvConfig = new ConfigStoreOptions().setType("env") .setConfig(new JsonObject().put("keys", - new JsonArray().add("DATA_DIR").add("TILE_SIZE").add("VXPORT").add("MIN_DS_POOL").add("MAX_DS_POOL"))); + new JsonArray().add("DATA_DIR").add("PROCESSED_DIR").add("TILE_SIZE").add("VXPORT").add("MIN_DS_POOL").add("MAX_DS_POOL"))); final ConfigRetrieverOptions myOptions = new ConfigRetrieverOptions().addStore(jsonEnvConfig); final ConfigRetriever myConfigRetriver = ConfigRetriever.create(vertx, myOptions); myConfigRetriver.getConfig(asyncResults -> System.out.println(asyncResults.result().encodePrettily())); @@ -102,6 +102,8 @@ public void start(final Promise startPromise) throws Exception { myConfigRetriver.getConfig(event -> { final var dataDirectoryString = event.result().getString("DATA_DIR", "."); final var dataDirectory = Path.of(dataDirectoryString).normalize().toAbsolutePath().normalize(); + final var processedDirectoryString = event.result().getString("PROCESSED_DIR", dataDirectory.resolve("processed").toString()); + final var processedDirectory = Path.of(processedDirectoryString).normalize().toAbsolutePath().normalize(); final var tileSize = event.result().getInteger("TILE_SIZE", 256); final var minDSPool = event.result().getInteger("MIN_DS_POOL", 4); final var maxDSPool = event.result().getInteger("MAX_DS_POOL", 16); @@ -111,6 +113,7 @@ public void start(final Promise startPromise) throws Exception { log.info("Trying to write configuration to local map"); final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); map.put("dataDirectory", new ShareableWrappers.PathWrapper(dataDirectory)); + map.put("processedDirectory", new ShareableWrappers.PathWrapper(processedDirectory)); map.put("tileSize", tileSize); map.put("VXPORT", port); map.put("MIN_DS_POOL", minDSPool); @@ -133,6 +136,7 @@ public void start(final Promise startPromise) throws Exception { } log.info("Using " + dataDirectory + " as data directory"); + log.info("Using " + processedDirectory + " as processed directory"); log.info("Using tile size " + tileSize); log.info("Server will start on port " + port); try { diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java index 48b63ba..45c8d2c 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java @@ -119,7 +119,11 @@ public void addHandlersToRouter(final @NotNull Router router) { map.put("openedFilename", filename); map.put("TileStatisticHolder", TileStatisticHolder.newDefaultStatisticHolder(chunkedFile.getResolutions().length)); - map.put("Track1DManager", new ShareableWrappers.Track1DManagerWrapper(new Track1DManager(dataDirectory))); + final var processedDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("processedDirectory"); + final var processedDirectory = processedDirectoryWrapper != null + ? processedDirectoryWrapper.getPath() + : dataDirectory.resolve("processed").normalize().toAbsolutePath(); + map.put("Track1DManager", new ShareableWrappers.Track1DManagerWrapper(new Track1DManager(dataDirectory, processedDirectory))); map.put("openProgress", new io.vertx.core.json.JsonObject() .put("stage", "done") diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java index 51678e9..2e30581 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java @@ -25,22 +25,35 @@ package ru.itmo.ctlab.hict.hict_server.handlers.tracks; import io.vertx.core.Vertx; +import io.vertx.core.WorkerExecutor; import io.vertx.core.json.Json; import io.vertx.core.shareddata.LocalMap; import io.vertx.ext.web.Router; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; +import ru.itmo.ctlab.hict.hict_library.domain.QueryLengthUnit; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; import java.util.Map; +import java.util.concurrent.TimeUnit; -@RequiredArgsConstructor public class TrackHandlersHolder extends HandlersHolder { private final Vertx vertx; + private final WorkerExecutor trackQueryWorkerExecutor; + + public TrackHandlersHolder(final Vertx vertx) { + this.vertx = vertx; + final var workerPoolSize = Math.max(2, Math.min(8, Runtime.getRuntime().availableProcessors() / 2)); + this.trackQueryWorkerExecutor = vertx.createSharedWorkerExecutor( + "hict-tracks-query-worker", + workerPoolSize, + 10, + TimeUnit.MINUTES + ); + } @Override public void addHandlersToRouter(final @NotNull Router router) { @@ -70,6 +83,12 @@ public void addHandlersToRouter(final @NotNull Router router) { request.getString("name"), request.getString("color") ); + final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFile = extractChunkedFile(map, ctx); + if (chunkedFile == null) { + return; + } + manager.startPrecompute(chunkedFile, summary.getTrackId(), false); ctx.response() .putHeader("content-type", "application/json") .end(Json.encode(summary)); @@ -126,10 +145,39 @@ public void addHandlersToRouter(final @NotNull Router router) { .end(Json.encode(Map.of("status", "removed", "trackId", trackId))); }); - router.post("/tracks/query_1d").blockingHandler(ctx -> { + router.post("/tracks/precompute/status").blockingHandler(ctx -> { + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(manager.getPrecomputeStatus())); + }); + + router.post("/tracks/precompute/start").blockingHandler(ctx -> { + final var request = ctx.body().asJsonObject(); + final var manager = getTrackManager(ctx); + if (manager == null) { + return; + } + final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFile = extractChunkedFile(map, ctx); + if (chunkedFile == null) { + return; + } + final var status = manager.startPrecompute( + chunkedFile, + request.getString("trackId"), + request.getBoolean("force", false) + ); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(status)); + }); + + router.post("/tracks/query_1d").handler(ctx -> { final var request = ctx.body().asJsonObject(); - final var startPx = request.getLong("startPx", 0L); - final var endPx = request.getLong("endPx", startPx + 1L); final var widthPx = request.getInteger("widthPx", 512); final var bpResolution = request.getLong("bpResolution", 1L); @@ -142,13 +190,77 @@ public void addHandlersToRouter(final @NotNull Router router) { if (chunkedFile == null) { return; } - final var result = manager.queryVisibleTracks(chunkedFile, startPx, endPx, widthPx, bpResolution); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(result)); + final var resolvedUnits = resolveUnits(request); + final var start = resolveStart(request, resolvedUnits); + final var end = resolveEnd(request, resolvedUnits, start + 1L); + this.trackQueryWorkerExecutor.executeBlocking(promise -> { + try { + promise.complete(manager.queryVisibleTracks(chunkedFile, start, end, widthPx, bpResolution, resolvedUnits)); + } catch (final Throwable t) { + promise.fail(t); + } + }, + false, + ar -> { + if (ar.failed()) { + ctx.fail(ar.cause()); + return; + } + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(ar.result())); + }); }); } + private static @NotNull QueryLengthUnit resolveUnits(final @NotNull io.vertx.core.json.JsonObject request) { + final var declared = request.getString("unit", request.getString("units")); + if (declared != null && !declared.isBlank()) { + return parseUnits(declared); + } + if (request.containsKey("startPx") || request.containsKey("endPx")) { + return QueryLengthUnit.PIXELS; + } + if (request.containsKey("startBin") || request.containsKey("endBin")) { + return QueryLengthUnit.BINS; + } + if (request.containsKey("startBP") || request.containsKey("endBP")) { + return QueryLengthUnit.BASE_PAIRS; + } + return QueryLengthUnit.PIXELS; + } + + private static long resolveStart(final @NotNull io.vertx.core.json.JsonObject request, + final @NotNull QueryLengthUnit units) { + return switch (units) { + case PIXELS -> request.getLong("startPx", request.getLong("start", 0L)); + case BINS -> request.getLong("startBin", request.getLong("start", 0L)); + case BASE_PAIRS -> request.getLong("startBP", request.getLong("start", 0L)); + }; + } + + private static long resolveEnd(final @NotNull io.vertx.core.json.JsonObject request, + final @NotNull QueryLengthUnit units, + final long fallback) { + return switch (units) { + case PIXELS -> request.getLong("endPx", request.getLong("end", fallback)); + case BINS -> request.getLong("endBin", request.getLong("end", fallback)); + case BASE_PAIRS -> request.getLong("endBP", request.getLong("end", fallback)); + }; + } + + private static @NotNull QueryLengthUnit parseUnits(final @NotNull String rawValue) { + final var normalized = rawValue.trim().toUpperCase(); + return switch (normalized) { + case "PIXEL", "PIXELS", "PX" -> QueryLengthUnit.PIXELS; + case "BIN", "BINS" -> QueryLengthUnit.BINS; + case "BP", "BASE_PAIRS", "BASEPAIR", "BASEPAIRS" -> QueryLengthUnit.BASE_PAIRS; + default -> throw new IllegalArgumentException( + "Unsupported query unit '" + rawValue + "'. Use one of: PIXELS, BINS, BP." + ); + }; + } + private Track1DManager getTrackManager(final @NotNull io.vertx.ext.web.RoutingContext ctx) { final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); final var managerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java index 7adda3e..dc394ee 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java @@ -24,6 +24,7 @@ package ru.itmo.ctlab.hict.hict_server.tracks; +import ch.systemsx.cisd.hdf5.HDF5Factory; import htsjdk.samtools.SAMRecord; import htsjdk.samtools.SAMSequenceDictionary; import htsjdk.samtools.SamReader; @@ -39,18 +40,26 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; +import ru.itmo.ctlab.hict.hict_library.chunkedfile.resolution.ResolutionDescriptor; import ru.itmo.ctlab.hict.hict_library.domain.ContigDirection; import ru.itmo.ctlab.hict.hict_library.domain.ContigHideType; +import ru.itmo.ctlab.hict.hict_library.domain.QueryLengthUnit; import ru.itmo.ctlab.hict.hict_library.trees.ContigTree; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.*; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -66,15 +75,44 @@ public class Track1DManager { "#edc948", "#b07aa1", "#ff9da7", "#9c755f", "#bab0ab" ); private static final int MAX_FEATURES_PER_QUERY = 250_000; + private static final String PRECOMPUTE_CACHE_VERSION = "1"; + private static final long MAX_PRECOMPUTE_VISIBLE_PIXELS = 2_000_000L; + private static final int PRECOMPUTE_EXECUTOR_THREADS = 1; + private static final long PRECOMPUTE_STATUS_TTL_MS = 15L * 60_000L; private final @NotNull Path dataDirectory; + private final @NotNull Path processedDirectory; private final @NotNull ReadWriteLock lock = new ReentrantReadWriteLock(); private final @NotNull LinkedHashMap tracks = new LinkedHashMap<>(); private final @NotNull AtomicLong trackCounter = new AtomicLong(0L); + private final @NotNull ExecutorService precomputeExecutor; + private final @NotNull ConcurrentHashMap precomputedSeriesCache = new ConcurrentHashMap<>(); + private final @NotNull ConcurrentHashMap precomputeRuntimeByTrackId = new ConcurrentHashMap<>(); private volatile @NotNull Map linkedFastaAliasesBySource = Map.of(); public Track1DManager(final @NotNull Path dataDirectory) { + this(dataDirectory, dataDirectory.resolve("processed")); + } + + public Track1DManager(final @NotNull Path dataDirectory, final @Nullable Path processedDirectory) { this.dataDirectory = dataDirectory.normalize().toAbsolutePath(); + this.processedDirectory = (processedDirectory == null ? this.dataDirectory.resolve("processed") : processedDirectory) + .normalize() + .toAbsolutePath(); + try { + Files.createDirectories(this.processedDirectory); + } catch (final IOException e) { + throw new RuntimeException("Failed to create processed directory " + this.processedDirectory, e); + } + this.precomputeExecutor = Executors.newFixedThreadPool( + PRECOMPUTE_EXECUTOR_THREADS, + r -> { + final var t = new Thread(r); + t.setDaemon(true); + t.setName("hict-track-precompute"); + return t; + } + ); } public @NotNull List listTrackFiles() { @@ -178,6 +216,8 @@ public void removeTrack(final @NotNull String trackId) { if (removed != null) { closeDataSourceQuietly(removed.dataSource()); } + this.precomputeRuntimeByTrackId.remove(trackId); + this.precomputedSeriesCache.keySet().removeIf(key -> key.trackId().equals(trackId)); } finally { this.lock.writeLock().unlock(); } @@ -188,15 +228,90 @@ public void close() { this.lock.writeLock().lock(); this.tracks.values().forEach(track -> closeDataSourceQuietly(track.dataSource())); this.tracks.clear(); + this.precomputeRuntimeByTrackId.clear(); + this.precomputedSeriesCache.clear(); } finally { this.lock.writeLock().unlock(); } + this.precomputeExecutor.shutdownNow(); } public void setLinkedFastaAliasesBySource(final @Nullable Map aliases) { this.linkedFastaAliasesBySource = aliases == null ? Map.of() : Map.copyOf(aliases); } + public @NotNull TracksPrecomputeStatus getPrecomputeStatus() { + final var now = System.currentTimeMillis(); + this.precomputeRuntimeByTrackId.entrySet().removeIf(entry -> + now - entry.getValue().lastUpdatedMs() > PRECOMPUTE_STATUS_TTL_MS + ); + final var statuses = this.precomputeRuntimeByTrackId.values().stream() + .sorted(Comparator.comparing(TrackPrecomputeRuntime::trackName)) + .map(TrackPrecomputeRuntime::toStatus) + .toList(); + final var runningCount = statuses.stream() + .filter(status -> "queued".equals(status.getStatus()) || "running".equals(status.getStatus())) + .count(); + return new TracksPrecomputeStatus(statuses, (int) runningCount, this.processedDirectory.toString()); + } + + public @NotNull TracksPrecomputeStatus startPrecompute(final @NotNull ChunkedFile chunkedFile, + final @Nullable String trackId, + final boolean force) { + final List selectedTracks; + try { + this.lock.readLock().lock(); + if (trackId != null && !trackId.isBlank()) { + final var state = this.tracks.get(trackId); + if (state == null) { + throw new IllegalArgumentException("Unknown track id " + trackId); + } + selectedTracks = List.of(state); + } else { + selectedTracks = this.tracks.values().stream().toList(); + } + } finally { + this.lock.readLock().unlock(); + } + + for (final var state : selectedTracks) { + scheduleTrackPrecompute(chunkedFile, state, force); + } + return getPrecomputeStatus(); + } + + public @NotNull QueryResult queryVisibleTracks(final @NotNull ChunkedFile chunkedFile, + final long start, + final long end, + final int widthPx, + final long bpResolution, + final @NotNull QueryLengthUnit units) { + final var safeWidth = Math.max(1, widthPx); + final var segmentsBuildResult = + buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource, bpResolution); + final var totalVisiblePixels = segmentsBuildResult.totalVisiblePixels(); + if (totalVisiblePixels <= 0L) { + return new QueryResult(0L, 1L, 0L, 1L, safeWidth, bpResolution, List.of()); + } + final var queryPxRange = resolveQueryPxRange( + chunkedFile, + start, + end, + bpResolution, + units, + segmentsBuildResult.orderedSegments(), + totalVisiblePixels + ); + return queryVisibleTracksInternal( + chunkedFile, + segmentsBuildResult, + queryPxRange.startPx(), + queryPxRange.endPx(), + safeWidth, + bpResolution + ); + } + public @NotNull QueryResult queryVisibleTracks(final @NotNull ChunkedFile chunkedFile, final long startPx, final long endPx, @@ -211,8 +326,18 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al } final var queryStartPx = Math.max(0L, Math.min(Math.min(startPx, endPx), totalVisiblePixels - 1L)); final var queryEndPx = Math.max(queryStartPx + 1L, Math.min(Math.max(startPx, endPx), totalVisiblePixels)); + return queryVisibleTracksInternal(chunkedFile, segmentsBuildResult, queryStartPx, queryEndPx, safeWidth, bpResolution); + } + + private @NotNull QueryResult queryVisibleTracksInternal(final @NotNull ChunkedFile chunkedFile, + final @NotNull SegmentBuildResult segmentsBuildResult, + final long queryStartPx, + final long queryEndPx, + final int safeWidth, + final long bpResolution) { final Map> sourceToAssemblySegments = segmentsBuildResult.sourceToAssemblySegments(); + final var assemblySignature = computeAssemblySignature(segmentsBuildResult.orderedSegments(), bpResolution); final List trackRenders = new ArrayList<>(); try { this.lock.readLock().lock(); @@ -220,7 +345,23 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al .filter(track -> track.visible) .forEach(track -> { try { - trackRenders.add(track.query(sourceToAssemblySegments, queryStartPx, queryEndPx, safeWidth, bpResolution)); + maybeScheduleTrackPrecomputeFromQuery(chunkedFile, track); + final var maybePrecomputed = getPrecomputedBinsIfReady( + chunkedFile, + track, + sourceToAssemblySegments, + segmentsBuildResult.orderedSegments(), + queryStartPx, + queryEndPx, + safeWidth, + bpResolution, + assemblySignature + ); + if (maybePrecomputed != null) { + trackRenders.add(maybePrecomputed); + } else { + trackRenders.add(track.query(sourceToAssemblySegments, queryStartPx, queryEndPx, safeWidth, bpResolution)); + } } catch (final RuntimeException ex) { final var message = ex.getMessage() != null ? ex.getMessage() : ex.getClass().getSimpleName(); log.error("Failed to query 1D track {} ({})", track.name(), track.trackId(), ex); @@ -239,6 +380,432 @@ public void setLinkedFastaAliasesBySource(final @Nullable Map al return new QueryResult(startBp, endBp, queryStartPx, queryEndPx, safeWidth, bpResolution, trackRenders); } + private static @NotNull QueryPxRange resolveQueryPxRange(final @NotNull ChunkedFile chunkedFile, + final long start, + final long end, + final long bpResolution, + final @NotNull QueryLengthUnit units, + final @NotNull List orderedSegments, + final long totalVisiblePixels) { + final var resolutionOrder = chunkedFile.getResolutionToIndex().get(bpResolution); + if (resolutionOrder == null) { + throw new IllegalArgumentException("Unsupported resolution for 1D track query: " + bpResolution); + } + final var minCoord = Math.min(start, end); + final var maxCoord = Math.max(start, end); + if (units == QueryLengthUnit.PIXELS) { + final var startPx = Math.max(0L, Math.min(minCoord, totalVisiblePixels - 1L)); + final var endPx = Math.max(startPx + 1L, Math.min(maxCoord, totalVisiblePixels)); + return new QueryPxRange(startPx, endPx); + } + + final var bpResolutionDescriptor = ResolutionDescriptor.fromResolutionOrder(resolutionOrder); + final var bpResolution0 = ResolutionDescriptor.fromResolutionOrder(0); + final var totalAssemblyBp = chunkedFile.getContigTree().getLengthInUnits(QueryLengthUnit.BASE_PAIRS, bpResolution0); + if (totalAssemblyBp <= 0L) { + return new QueryPxRange(0L, 1L); + } + + final long startBp; + final long endBpExcl; + if (units == QueryLengthUnit.BASE_PAIRS) { + startBp = Math.max(0L, Math.min(minCoord, totalAssemblyBp - 1L)); + endBpExcl = Math.max(startBp + 1L, Math.min(maxCoord, totalAssemblyBp)); + } else if (units == QueryLengthUnit.BINS) { + final var totalBins = chunkedFile.getContigTree().getLengthInUnits(QueryLengthUnit.BINS, bpResolutionDescriptor); + if (totalBins <= 0L) { + return new QueryPxRange(0L, 1L); + } + final var startBin = Math.max(0L, Math.min(minCoord, totalBins - 1L)); + final var endBinExcl = Math.max(startBin + 1L, Math.min(maxCoord, totalBins)); + startBp = chunkedFile.convertUnits( + startBin, + bpResolutionDescriptor, + QueryLengthUnit.BINS, + bpResolution0, + QueryLengthUnit.BASE_PAIRS + ); + if (endBinExcl >= totalBins) { + endBpExcl = totalAssemblyBp; + } else { + endBpExcl = chunkedFile.convertUnits( + endBinExcl, + bpResolutionDescriptor, + QueryLengthUnit.BINS, + bpResolution0, + QueryLengthUnit.BASE_PAIRS + ); + } + } else { + throw new IllegalArgumentException("Unsupported query units for 1D track query: " + units); + } + + final var clampedStartBp = Math.max(0L, Math.min(startBp, totalAssemblyBp - 1L)); + final var clampedEndBpExcl = Math.max(clampedStartBp + 1L, Math.min(endBpExcl, totalAssemblyBp)); + final var startPx = mapAssemblyBpToVisiblePx(clampedStartBp, orderedSegments, bpResolution, totalVisiblePixels); + final var endPx = Math.min( + totalVisiblePixels, + mapAssemblyBpToVisiblePx(Math.max(clampedStartBp, clampedEndBpExcl - 1L), orderedSegments, bpResolution, totalVisiblePixels) + 1L + ); + final var safeEndPx = Math.max(startPx + 1L, endPx); + return new QueryPxRange(startPx, safeEndPx); + } + + private void maybeScheduleTrackPrecomputeFromQuery(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track) { + final var runtime = this.precomputeRuntimeByTrackId.get(track.trackId()); + if (runtime != null && runtime.isActive()) { + return; + } + scheduleTrackPrecompute(chunkedFile, track, false); + } + + private void scheduleTrackPrecompute(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final boolean force) { + final var runtime = this.precomputeRuntimeByTrackId.computeIfAbsent( + track.trackId(), + ignored -> new TrackPrecomputeRuntime(track.trackId(), track.name()) + ); + synchronized (runtime) { + if (runtime.isActive()) { + return; + } + runtime.markQueued(); + } + this.precomputeExecutor.submit(() -> runTrackPrecompute(chunkedFile, track, force, runtime)); + } + + private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final boolean force, + final @NotNull TrackPrecomputeRuntime runtime) { + try { + final var tasks = buildPrecomputeTasks(chunkedFile, track, force); + runtime.setTotalTasks(tasks.size()); + if (tasks.isEmpty()) { + runtime.markFinished(); + return; + } + int completed = 0; + for (final var task : tasks) { + runtime.markRunning(task.bpResolution() + "bp/" + task.modeKey(), completed); + final var key = new PrecomputedSeriesKey(track.trackId(), task.bpResolution(), task.assemblySignature(), task.modeKey()); + PrecomputedSeries series = this.precomputedSeriesCache.get(key); + if (series == null || force) { + series = loadPrecomputedSeriesFromSidecar(task.sidecarFile(), task.totalVisiblePixels()).orElse(null); + } + if (series == null || force) { + series = computePrecomputedSeries(chunkedFile, track, task); + persistPrecomputedSeries(task.sidecarFile(), series); + } + this.precomputedSeriesCache.put(key, series); + completed++; + runtime.markTaskDone(completed); + } + runtime.markFinished(); + } catch (final Exception ex) { + runtime.markFailed(ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage()); + log.error("Failed to precompute track {}", track.trackId(), ex); + } + } + + private @NotNull List buildPrecomputeTasks(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final boolean force) { + final var tasks = new ArrayList(); + final var resolutions = Arrays.stream(chunkedFile.getResolutions()).boxed().sorted(Comparator.reverseOrder()).toList(); + final var modeKeys = modeKeysForTrack(track); + for (final var bpResolution : resolutions) { + final var segmentsBuildResult = buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource, bpResolution); + final var totalVisiblePixels = segmentsBuildResult.totalVisiblePixels(); + if (totalVisiblePixels <= 0L || totalVisiblePixels > MAX_PRECOMPUTE_VISIBLE_PIXELS) { + continue; + } + final var assemblySignature = computeAssemblySignature(segmentsBuildResult.orderedSegments(), bpResolution); + for (final var modeKey : modeKeys) { + final var key = new PrecomputedSeriesKey(track.trackId(), bpResolution, assemblySignature, modeKey); + if (!force && this.precomputedSeriesCache.containsKey(key)) { + continue; + } + final var sidecarPath = sidecarPathForVector(chunkedFile, track, bpResolution, assemblySignature, modeKey); + if (!force && Files.exists(sidecarPath)) { + continue; + } + tasks.add(new PrecomputeTask(bpResolution, modeKey, totalVisiblePixels, assemblySignature, sidecarPath)); + } + } + return tasks; + } + + private @Nullable TrackRender getPrecomputedBinsIfReady(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final @NotNull Map> sourceToAssemblySegments, + final @NotNull List orderedSegments, + final long queryStartPx, + final long queryEndPx, + final int widthPx, + final long bpResolution, + final @NotNull String assemblySignature) { + final var totalVisiblePixels = orderedSegments.isEmpty() ? 0L : orderedSegments.get(orderedSegments.size() - 1).visiblePxEnd(); + if (totalVisiblePixels <= 0L || totalVisiblePixels > MAX_PRECOMPUTE_VISIBLE_PIXELS) { + return null; + } + final var modeKey = activeModeKey(track); + final var key = new PrecomputedSeriesKey(track.trackId(), bpResolution, assemblySignature, modeKey); + var series = this.precomputedSeriesCache.get(key); + if (series == null) { + final var sidecar = sidecarPathForVector(chunkedFile, track, bpResolution, assemblySignature, modeKey); + series = loadPrecomputedSeriesFromSidecar(sidecar, totalVisiblePixels).orElse(null); + if (series != null) { + this.precomputedSeriesCache.put(key, series); + } + } + if (series == null) { + return null; + } + final var strategy = aggregationStrategy(track); + final var bins = aggregatePrecomputedSeries(series, queryStartPx, queryEndPx, widthPx, strategy); + final var maxValue = bins.stream().mapToDouble(TrackBin::getValue).max().orElse(0.0d); + return new TrackRender(track.trackId(), track.name(), track.type().name(), track.color(), bins, maxValue, null); + } + + private @NotNull PrecomputedSeries computePrecomputedSeries(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final @NotNull PrecomputeTask task) { + final var segmentsBuildResult = buildSourceToAssemblySegments(chunkedFile, this.linkedFastaAliasesBySource, task.bpResolution()); + final var totalVisiblePixels = (int) Math.max(1L, Math.min(Integer.MAX_VALUE, task.totalVisiblePixels())); + final var bamRenderMode = bamModeForKey(track.bamRenderMode(), task.modeKey()); + final var bigWigAggregationMode = bigWigModeForKey(track.bigWigAggregationMode(), task.modeKey()); + final var bins = queryBinsForTrack( + track.type(), + track.dataSource(), + segmentsBuildResult.sourceToAssemblySegments(), + 0L, + task.totalVisiblePixels(), + totalVisiblePixels, + task.bpResolution(), + bamRenderMode, + bigWigAggregationMode + ); + final var values = new double[totalVisiblePixels]; + final var support = new long[totalVisiblePixels]; + for (final var bin : bins) { + final var rawStart = bin.getStartPx() == null ? bin.getStartBp() : bin.getStartPx(); + final var rawEnd = bin.getEndPx() == null ? bin.getEndBp() : bin.getEndPx(); + final int start = (int) Math.max(0L, Math.min(rawStart, totalVisiblePixels - 1L)); + final int end = (int) Math.max(start + 1L, Math.min(rawEnd, totalVisiblePixels)); + for (int i = start; i < end; i++) { + values[i] = bin.getValue(); + support[i] = Math.max(support[i], Math.max(1L, bin.getCount())); + } + } + return new PrecomputedSeries(values, support); + } + + private @NotNull List aggregatePrecomputedSeries(final @NotNull PrecomputedSeries series, + final long queryStartPx, + final long queryEndPx, + final int widthPx, + final @NotNull PrecomputeAggregationStrategy strategy) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEndPx - queryStartPx); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + final var bins = new ArrayList(bucketCount); + for (int i = 0; i < bucketCount; i++) { + final var startPx = queryStartPx + (long) Math.floor(i * bucketSpan); + final var endPx = Math.min(queryEndPx, queryStartPx + (long) Math.ceil((i + 1) * bucketSpan)); + final var safeEndPx = Math.max(startPx + 1L, endPx); + final int from = (int) Math.max(0L, Math.min(startPx, series.values().length - 1L)); + final int to = (int) Math.max(from + 1L, Math.min(safeEndPx, series.values().length)); + double maxValue = 0.0d; + double sumValue = 0.0d; + long supportSum = 0L; + long supportCount = 0L; + for (int idx = from; idx < to; idx++) { + final var value = series.values()[idx]; + final var support = series.support()[idx]; + maxValue = Math.max(maxValue, value); + sumValue += value; + supportSum += support; + if (support > 0L) { + supportCount++; + } + } + final double value = switch (strategy) { + case MAX -> maxValue; + case MEAN_ALL_PIXELS -> sumValue / Math.max(1.0d, to - from); + case MEAN_PRESENT_PIXELS -> supportCount > 0L ? (sumValue / supportCount) : 0.0d; + case SUM -> sumValue; + }; + if (supportSum <= 0L && value <= 0.0d) { + continue; + } + bins.add(new TrackBin(startPx, safeEndPx, value, Math.max(1L, supportSum), null, startPx, safeEndPx)); + } + return bins; + } + + private @NotNull Optional loadPrecomputedSeriesFromSidecar(final @NotNull Path sidecarPath, + final long expectedLength) { + if (!Files.exists(sidecarPath) || !Files.isRegularFile(sidecarPath)) { + return Optional.empty(); + } + try (final var reader = HDF5Factory.openForReading(sidecarPath.toFile())) { + if (!reader.object().isDataSet("/cache/values") || !reader.object().isDataSet("/cache/support")) { + return Optional.empty(); + } + final var valuesDims = reader.object().getDataSetInformation("/cache/values").getDimensions(); + if (valuesDims.length != 1 || valuesDims[0] != expectedLength) { + return Optional.empty(); + } + final var values = reader.float64().readArray("/cache/values"); + final var support = reader.int64().readArray("/cache/support"); + if (values.length != support.length) { + return Optional.empty(); + } + return Optional.of(new PrecomputedSeries(values, support)); + } catch (final Exception e) { + log.debug("Could not load sidecar precompute {}: {}", sidecarPath, e.getMessage()); + return Optional.empty(); + } + } + + private void persistPrecomputedSeries(final @NotNull Path sidecarPath, + final @NotNull PrecomputedSeries series) { + try { + Files.createDirectories(sidecarPath.getParent()); + final var tmpPath = sidecarPath.resolveSibling(sidecarPath.getFileName() + ".tmp"); + try (final var writer = HDF5Factory.open(tmpPath.toFile())) { + if (!writer.object().isGroup("/cache")) { + writer.object().createGroup("/cache"); + } + writer.string().setAttr("/cache", "version", PRECOMPUTE_CACHE_VERSION); + writer.int64().setAttr("/cache", "length", series.values().length); + writer.float64().writeArray("/cache/values", series.values()); + writer.int64().writeArray("/cache/support", series.support()); + } + Files.move(tmpPath, sidecarPath, StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception e) { + log.warn("Failed to write precomputed sidecar {}", sidecarPath, e); + } + } + + private @NotNull Path sidecarPathForVector(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track, + final long bpResolution, + final @NotNull String assemblySignature, + final @NotNull String modeKey) { + final var trackSource = resolveDataPath(track.sourceFile()); + final var hictPath = chunkedFile.getHdfFilePath(); + final String fingerprint; + try { + fingerprint = String.join("|", + PRECOMPUTE_CACHE_VERSION, + trackSource.toString(), + String.valueOf(Files.size(trackSource)), + String.valueOf(Files.getLastModifiedTime(trackSource).toMillis()), + hictPath.toString(), + String.valueOf(Files.size(hictPath)), + String.valueOf(Files.getLastModifiedTime(hictPath).toMillis()), + String.valueOf(bpResolution), + assemblySignature, + modeKey + ); + } catch (final IOException e) { + throw new RuntimeException("Cannot build precompute fingerprint for " + track.sourceFile(), e); + } + final var fileName = sha256Hex(fingerprint) + ".h5"; + return this.processedDirectory.resolve("track_precompute").resolve(fileName); + } + + private static @NotNull String computeAssemblySignature(final @NotNull List orderedSegments, + final long bpResolution) { + long hash = 1469598103934665603L; + hash = fnv1a(hash, bpResolution); + hash = fnv1a(hash, orderedSegments.size()); + for (final var segment : orderedSegments) { + hash = fnv1a(hash, segment.sourceStart()); + hash = fnv1a(hash, segment.sourceEnd()); + hash = fnv1a(hash, segment.assemblyStart()); + hash = fnv1a(hash, segment.assemblyEnd()); + hash = fnv1a(hash, segment.visiblePxStart()); + hash = fnv1a(hash, segment.visiblePxEnd()); + hash = fnv1a(hash, segment.reversed() ? 1L : 0L); + } + return Long.toUnsignedString(hash, 16); + } + + private static long fnv1a(long hash, final long value) { + hash ^= value; + hash *= 1099511628211L; + return hash; + } + + private static @NotNull String sha256Hex(final @NotNull String input) { + try { + final var digest = MessageDigest.getInstance("SHA-256").digest(input.getBytes(StandardCharsets.UTF_8)); + final var sb = new StringBuilder(digest.length * 2); + for (final byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (final NoSuchAlgorithmException e) { + throw new RuntimeException("SHA-256 is not available", e); + } + } + + private static @NotNull List modeKeysForTrack(final @NotNull TrackState track) { + return switch (track.type()) { + case BIGWIG -> List.of("MAX", "MEAN", "SUM"); + case BAM -> List.of("COVERAGE", "READ_DENSITY"); + case BED, VCF -> List.of("DEFAULT"); + case UNSUPPORTED -> List.of("DEFAULT"); + }; + } + + private static @NotNull String activeModeKey(final @NotNull TrackState track) { + return switch (track.type()) { + case BIGWIG -> track.bigWigAggregationMode().name(); + case BAM -> track.bamRenderMode().name(); + case BED, VCF, UNSUPPORTED -> "DEFAULT"; + }; + } + + private static @NotNull BamRenderMode bamModeForKey(final @NotNull BamRenderMode fallback, + final @NotNull String modeKey) { + return switch (modeKey) { + case "READ_DENSITY" -> BamRenderMode.READ_DENSITY; + case "COVERAGE" -> BamRenderMode.COVERAGE; + default -> fallback; + }; + } + + private static @NotNull BigWigAggregationMode bigWigModeForKey(final @NotNull BigWigAggregationMode fallback, + final @NotNull String modeKey) { + return switch (modeKey) { + case "MAX" -> BigWigAggregationMode.MAX; + case "MEAN" -> BigWigAggregationMode.MEAN; + case "SUM" -> BigWigAggregationMode.SUM; + default -> fallback; + }; + } + + private static @NotNull PrecomputeAggregationStrategy aggregationStrategy(final @NotNull TrackState track) { + return switch (track.type()) { + case BIGWIG -> switch (track.bigWigAggregationMode()) { + case MAX -> PrecomputeAggregationStrategy.MAX; + case MEAN -> PrecomputeAggregationStrategy.MEAN_PRESENT_PIXELS; + case SUM -> PrecomputeAggregationStrategy.MEAN_ALL_PIXELS; + }; + case BAM -> switch (track.bamRenderMode()) { + case COVERAGE -> PrecomputeAggregationStrategy.MEAN_ALL_PIXELS; + case READ_DENSITY -> PrecomputeAggregationStrategy.SUM; + }; + case BED, VCF, UNSUPPORTED -> PrecomputeAggregationStrategy.MAX; + }; + } + private @NotNull SegmentBuildResult buildSourceToAssemblySegments(final @NotNull ChunkedFile chunkedFile, final @NotNull Map linkedFastaAliasesBySource, final long bpResolution) { @@ -453,6 +1020,38 @@ private static long mapVisiblePxToAssemblyBp(final long px, return localPxToAssemblyBp(segment, localPx, bpResolution); } + private static long mapAssemblyBpToVisiblePx(final long assemblyBp, + final @NotNull List orderedSegments, + final long bpResolution, + final long totalVisiblePixels) { + if (orderedSegments.isEmpty() || totalVisiblePixels <= 0L) { + return 0L; + } + if (assemblyBp <= orderedSegments.get(0).assemblyStart()) { + return 0L; + } + if (assemblyBp >= orderedSegments.get(orderedSegments.size() - 1).assemblyEnd()) { + return totalVisiblePixels; + } + int lo = 0; + int hi = orderedSegments.size(); + while (lo < hi) { + final int mid = (lo + hi) >>> 1; + if (orderedSegments.get(mid).assemblyEnd() <= assemblyBp) { + lo = mid + 1; + } else { + hi = mid; + } + } + final int idx = Math.max(0, Math.min(lo, orderedSegments.size() - 1)); + final var segment = orderedSegments.get(idx); + if (assemblyBp < segment.assemblyStart()) { + return segment.visiblePxStart(); + } + final var localPx = assemblyBpToLocalPx(segment, assemblyBp, bpResolution); + return Math.max(0L, Math.min(totalVisiblePixels, segment.visiblePxStart() + localPx)); + } + private static @NotNull Optional mapVisiblePxIntervalToSegmentSource(final @NotNull AssemblySegment segment, final long queryStartPx, final long queryEndPx, @@ -817,6 +1416,48 @@ private static void accumulateReadDensityValue(final long featureStart, .toList(); } + private static @NotNull List queryBinsForTrack(final @NotNull TrackType type, + final @NotNull TrackDataSource dataSource, + final @NotNull Map> sourceToAssemblySegments, + final long queryStartPx, + final long queryEndPx, + final int widthPx, + final long bpResolution, + final @NotNull BamRenderMode bamRenderMode, + final @NotNull BigWigAggregationMode bigWigAggregationMode) { + if (type == TrackType.BAM && dataSource instanceof BamTrackDataSource bamDataSource) { + return bamDataSource.queryBins( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution, + bamRenderMode + ); + } + if (type == TrackType.BIGWIG && dataSource instanceof BigWigTrackDataSource bigWigDataSource) { + return bigWigDataSource.queryBins( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution, + bigWigAggregationMode + ); + } + final var projectedFeatures = dataSource.projectFeatures( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + bpResolution + ); + final var maxFeatureCount = Math.max(widthPx * 8, 8192); + if (projectedFeatures.size() > maxFeatureCount) { + return aggregateFeatures(projectedFeatures, queryStartPx, queryEndPx, widthPx); + } + return toBins(projectedFeatures); + } + private enum TrackType { BED, VCF, @@ -1397,6 +2038,33 @@ private static int clampToInt(final long value) { private record SourceInterval(long start, long end) { } + private record QueryPxRange(long startPx, long endPx) { + } + + private record PrecomputeTask(long bpResolution, + @NotNull String modeKey, + long totalVisiblePixels, + @NotNull String assemblySignature, + @NotNull Path sidecarFile) { + } + + private record PrecomputedSeriesKey(@NotNull String trackId, + long bpResolution, + @NotNull String assemblySignature, + @NotNull String modeKey) { + } + + private record PrecomputedSeries(double @NotNull [] values, + long @NotNull [] support) { + } + + private enum PrecomputeAggregationStrategy { + MAX, + MEAN_ALL_PIXELS, + MEAN_PRESENT_PIXELS, + SUM + } + private record SegmentBuildResult(@NotNull Map> sourceToAssemblySegments, @NotNull List orderedSegments, long totalVisiblePixels) { @@ -1422,6 +2090,118 @@ private record ProjectedFeature(long startBp, String label) { } + @Getter + @RequiredArgsConstructor + public static final class TrackPrecomputeStatus { + private final @NotNull String trackId; + private final @NotNull String trackName; + private final @NotNull String status; + private final int totalTasks; + private final int completedTasks; + private final double progress; + private final @NotNull String currentTask; + private final String error; + private final long updatedAtMs; + } + + @Getter + @RequiredArgsConstructor + public static final class TracksPrecomputeStatus { + private final @NotNull List tracks; + private final int runningJobs; + private final @NotNull String processedDirectory; + } + + private static final class TrackPrecomputeRuntime { + private final String trackId; + private volatile String trackName; + private volatile String status; + private volatile int totalTasks; + private volatile int completedTasks; + private volatile String currentTask; + private volatile String error; + private volatile long lastUpdatedMs; + + private TrackPrecomputeRuntime(final @NotNull String trackId, final @NotNull String trackName) { + this.trackId = trackId; + this.trackName = trackName; + this.status = "idle"; + this.totalTasks = 0; + this.completedTasks = 0; + this.currentTask = ""; + this.error = null; + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private boolean isActive() { + return "queued".equals(this.status) || "running".equals(this.status); + } + + private void markQueued() { + this.status = "queued"; + this.currentTask = ""; + this.error = null; + this.completedTasks = 0; + this.totalTasks = 0; + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private void setTotalTasks(final int totalTasks) { + this.totalTasks = Math.max(0, totalTasks); + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private void markRunning(final @NotNull String currentTask, final int completedTasks) { + this.status = "running"; + this.currentTask = currentTask; + this.completedTasks = Math.max(0, completedTasks); + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private void markTaskDone(final int completedTasks) { + this.completedTasks = Math.max(0, completedTasks); + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private void markFinished() { + this.status = "finished"; + this.currentTask = ""; + this.error = null; + this.completedTasks = Math.max(this.completedTasks, this.totalTasks); + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private void markFailed(final @NotNull String error) { + this.status = "failed"; + this.error = error; + this.currentTask = ""; + this.lastUpdatedMs = System.currentTimeMillis(); + } + + private long lastUpdatedMs() { + return this.lastUpdatedMs; + } + + private String trackName() { + return this.trackName; + } + + private TrackPrecomputeStatus toStatus() { + final var progress = this.totalTasks > 0 ? Math.min(1.0d, this.completedTasks / (double) this.totalTasks) : 0.0d; + return new TrackPrecomputeStatus( + this.trackId, + this.trackName, + this.status, + this.totalTasks, + this.completedTasks, + progress, + this.currentTask == null ? "" : this.currentTask, + this.error, + this.lastUpdatedMs + ); + } + } + @Getter @RequiredArgsConstructor public static final class QueryResult { @@ -1553,39 +2333,17 @@ private TrackRender query(final @NotNull Map> sour final long queryEndPx, final int widthPx, final long bpResolution) { - final List bins; - if (type == TrackType.BAM && dataSource instanceof BamTrackDataSource bamDataSource) { - bins = bamDataSource.queryBins( - sourceToAssemblySegments, - queryStartPx, - queryEndPx, - widthPx, - bpResolution, - bamRenderMode - ); - } else if (type == TrackType.BIGWIG && dataSource instanceof BigWigTrackDataSource bigWigDataSource) { - bins = bigWigDataSource.queryBins( - sourceToAssemblySegments, - queryStartPx, - queryEndPx, - widthPx, - bpResolution, - bigWigAggregationMode - ); - } else { - final var projectedFeatures = dataSource.projectFeatures( - sourceToAssemblySegments, - queryStartPx, - queryEndPx, - bpResolution - ); - final var maxFeatureCount = Math.max(widthPx * 8, 8192); - if (projectedFeatures.size() > maxFeatureCount) { - bins = aggregateFeatures(projectedFeatures, queryStartPx, queryEndPx, widthPx); - } else { - bins = toBins(projectedFeatures); - } - } + final var bins = queryBinsForTrack( + type, + dataSource, + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution, + bamRenderMode, + bigWigAggregationMode + ); final var maxValue = bins.stream().mapToDouble(TrackBin::getValue).max().orElse(0.0d); return new TrackRender(trackId, name, type.name(), color, bins, maxValue, null); } diff --git a/version.txt b/version.txt index 6aa072e..f17dda7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.83-1ce0372-webui_1a02368 \ No newline at end of file +1.0.84-aa7deb2-webui_f21ba84 \ No newline at end of file From 8494e37a6d144a74b13e4491649acdd86a6a0c07 Mon Sep 17 00:00:00 2001 From: Alexander Serdyukov Date: Sat, 21 Mar 2026 04:37:00 +0400 Subject: [PATCH 4/5] Multilevel priority pools added --- .../ctlab/hict/hict_server/MainVerticle.java | 123 +++- .../concurrent/RequestTaskScheduler.java | 384 ++++++++++++ .../conversion/ConversionHandlersHolder.java | 473 ++++++++++----- .../handlers/fileop/FileOpHandlersHolder.java | 571 +++++++++++------- .../handlers/files/FSHandlersHolder.java | 154 +++-- .../names/NameMappingHandlersHolder.java | 172 ++++-- .../ScaffoldingOpHandlersHolder.java | 236 +++++--- .../handlers/tiles/TileHandlersHolder.java | 419 +++++++------ .../handlers/tracks/TrackHandlersHolder.java | 227 ++++--- .../util/shareable/ShareableWrappers.java | 7 + version.txt | 2 +- 11 files changed, 1895 insertions(+), 873 deletions(-) create mode 100644 src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java index 037b708..5b97479 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/MainVerticle.java @@ -49,6 +49,7 @@ import ru.itmo.ctlab.hict.hict_library.chunkedfile.hdf5.HDF5LibraryInitializer; import ru.itmo.ctlab.hict.hict_library.visualization.SimpleVisualizationOptions; import ru.itmo.ctlab.hict.hict_library.visualization.colormap.gradient.SimpleLinearGradient; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.handlers.fileop.FileOpHandlersHolder; import ru.itmo.ctlab.hict.hict_server.handlers.files.FSHandlersHolder; import ru.itmo.ctlab.hict.hict_server.handlers.names.NameMappingHandlersHolder; @@ -62,6 +63,7 @@ import java.awt.*; import java.nio.file.Path; import java.util.ArrayList; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.concurrent.BrokenBarrierException; @@ -69,6 +71,7 @@ @Slf4j(topic = "MainVerticle") public class MainVerticle extends AbstractVerticle { + private RequestTaskScheduler requestTaskScheduler; static { HDF5LibraryInitializer.initializeHDF5Library(); @@ -93,7 +96,26 @@ public void start(final Promise startPromise) throws Exception { final ConfigStoreOptions jsonEnvConfig = new ConfigStoreOptions().setType("env") .setConfig(new JsonObject().put("keys", - new JsonArray().add("DATA_DIR").add("PROCESSED_DIR").add("TILE_SIZE").add("VXPORT").add("MIN_DS_POOL").add("MAX_DS_POOL"))); + new JsonArray() + .add("DATA_DIR") + .add("PROCESSED_DIR") + .add("TILE_SIZE") + .add("VXPORT") + .add("MIN_DS_POOL") + .add("MAX_DS_POOL") + .add("HICT_WORKERS_TOTAL_MAX") + .add("HICT_WORKERS_QUEUE_CAPACITY") + .add("HICT_WORKERS_KEEPALIVE_SECONDS") + .add("HICT_WORKERS_UI_MIN") + .add("HICT_WORKERS_UI_MAX") + .add("HICT_WORKERS_ASSEMBLY_MIN") + .add("HICT_WORKERS_ASSEMBLY_MAX") + .add("HICT_WORKERS_TILE_MIN") + .add("HICT_WORKERS_TILE_MAX") + .add("HICT_WORKERS_TRACK_MIN") + .add("HICT_WORKERS_TRACK_MAX") + .add("HICT_WORKERS_EXPORT_MIN") + .add("HICT_WORKERS_EXPORT_MAX"))); final ConfigRetrieverOptions myOptions = new ConfigRetrieverOptions().addStore(jsonEnvConfig); final ConfigRetriever myConfigRetriver = ConfigRetriever.create(vertx, myOptions); myConfigRetriver.getConfig(asyncResults -> System.out.println(asyncResults.result().encodePrettily())); @@ -104,10 +126,54 @@ public void start(final Promise startPromise) throws Exception { final var dataDirectory = Path.of(dataDirectoryString).normalize().toAbsolutePath().normalize(); final var processedDirectoryString = event.result().getString("PROCESSED_DIR", dataDirectory.resolve("processed").toString()); final var processedDirectory = Path.of(processedDirectoryString).normalize().toAbsolutePath().normalize(); - final var tileSize = event.result().getInteger("TILE_SIZE", 256); - final var minDSPool = event.result().getInteger("MIN_DS_POOL", 4); - final var maxDSPool = event.result().getInteger("MAX_DS_POOL", 16); - final var port = event.result().getInteger("VXPORT", 5000); + final var tileSize = getIntegerSetting(event.result(), "TILE_SIZE", 256); + final var minDSPool = getIntegerSetting(event.result(), "MIN_DS_POOL", 4); + final var maxDSPool = getIntegerSetting(event.result(), "MAX_DS_POOL", 16); + final var port = getIntegerSetting(event.result(), "VXPORT", 5000); + final int cores = Math.max(2, Runtime.getRuntime().availableProcessors()); + final int totalWorkersDefault = Math.max(10, cores * 2); + final int totalWorkers = getIntegerSetting(event.result(), "HICT_WORKERS_TOTAL_MAX", totalWorkersDefault); + final int queueCapacity = getIntegerSetting(event.result(), "HICT_WORKERS_QUEUE_CAPACITY", 32); + final int keepAliveSeconds = getIntegerSetting(event.result(), "HICT_WORKERS_KEEPALIVE_SECONDS", 30); + final int defaultPoolMax = Math.max(2, Math.min(totalWorkers, cores)); + final var perPrioritySizing = new EnumMap( + RequestTaskScheduler.RequestPriority.class + ); + perPrioritySizing.put( + RequestTaskScheduler.RequestPriority.UI_UX, + new RequestTaskScheduler.PoolSizing( + getIntegerSetting(event.result(), "HICT_WORKERS_UI_MIN", 2), + getIntegerSetting(event.result(), "HICT_WORKERS_UI_MAX", defaultPoolMax) + ) + ); + perPrioritySizing.put( + RequestTaskScheduler.RequestPriority.ASSEMBLY, + new RequestTaskScheduler.PoolSizing( + getIntegerSetting(event.result(), "HICT_WORKERS_ASSEMBLY_MIN", 2), + getIntegerSetting(event.result(), "HICT_WORKERS_ASSEMBLY_MAX", defaultPoolMax) + ) + ); + perPrioritySizing.put( + RequestTaskScheduler.RequestPriority.TILE, + new RequestTaskScheduler.PoolSizing( + getIntegerSetting(event.result(), "HICT_WORKERS_TILE_MIN", 2), + getIntegerSetting(event.result(), "HICT_WORKERS_TILE_MAX", defaultPoolMax) + ) + ); + perPrioritySizing.put( + RequestTaskScheduler.RequestPriority.TRACK, + new RequestTaskScheduler.PoolSizing( + getIntegerSetting(event.result(), "HICT_WORKERS_TRACK_MIN", 2), + getIntegerSetting(event.result(), "HICT_WORKERS_TRACK_MAX", defaultPoolMax) + ) + ); + perPrioritySizing.put( + RequestTaskScheduler.RequestPriority.EXPORT, + new RequestTaskScheduler.PoolSizing( + getIntegerSetting(event.result(), "HICT_WORKERS_EXPORT_MIN", 2), + getIntegerSetting(event.result(), "HICT_WORKERS_EXPORT_MAX", defaultPoolMax) + ) + ); try { log.info("Trying to write configuration to local map"); @@ -118,6 +184,19 @@ public void start(final Promise startPromise) throws Exception { map.put("VXPORT", port); map.put("MIN_DS_POOL", minDSPool); map.put("MAX_DS_POOL", maxDSPool); + this.requestTaskScheduler = new RequestTaskScheduler( + vertx, + new RequestTaskScheduler.SchedulerConfig( + totalWorkers, + queueCapacity, + keepAliveSeconds, + perPrioritySizing + ) + ); + map.put( + RequestTaskScheduler.LOCAL_MAP_KEY, + new ShareableWrappers.RequestTaskSchedulerWrapper(this.requestTaskScheduler) + ); final var defaultVisualizationOptions = new SimpleVisualizationOptions(10.0, 0.0, false, false, false, new SimpleLinearGradient( @@ -219,4 +298,38 @@ public void start(final Promise startPromise) throws Exception { } }); } + + @Override + public void stop(final Promise stopPromise) { + if (this.requestTaskScheduler != null) { + this.requestTaskScheduler.close(); + this.requestTaskScheduler = null; + } + stopPromise.complete(); + } + + private static int getIntegerSetting(final @NotNull JsonObject config, + final @NotNull String key, + final int defaultValue) { + final Object raw = config.getValue(key); + if (raw instanceof Number number) { + return number.intValue(); + } + if (raw instanceof String value && !value.isBlank()) { + try { + return Integer.parseInt(value.trim()); + } catch (final NumberFormatException ignored) { + // Fall through to system property/default. + } + } + final String systemPropertyValue = System.getProperty(key); + if (systemPropertyValue != null && !systemPropertyValue.isBlank()) { + try { + return Integer.parseInt(systemPropertyValue.trim()); + } catch (final NumberFormatException ignored) { + // Fall through to default. + } + } + return defaultValue; + } } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java new file mode 100644 index 0000000..9abd9be --- /dev/null +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java @@ -0,0 +1,384 @@ +/* + * MIT License + * + * Copyright (c) 2021-2026. Aleksandr Serdiukov, Anton Zamyatin, Aleksandr Sinitsyn, Vitalii Dravgelis and Computer Technologies Laboratory ITMO University team. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package ru.itmo.ctlab.hict.hict_server.concurrent; + +import io.vertx.core.Vertx; +import io.vertx.ext.web.RoutingContext; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.EnumMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.FutureTask; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +@Slf4j +public final class RequestTaskScheduler implements AutoCloseable { + public static final String LOCAL_MAP_KEY = "requestTaskScheduler"; + + private final @NotNull Vertx vertx; + private final @NotNull EnumMap pools; + private final @NotNull EnumMap poolSizing; + private final @NotNull EnumMap activeWorkersByPriority; + private final @NotNull EnumMap generations; + private final @NotNull EnumMap>>> trackedTasks; + private final @NotNull Semaphore globalElasticPermits; + + @Getter + public static final class PoolSizing { + private final int minWorkers; + private final int maxWorkers; + + public PoolSizing(final int minWorkers, final int maxWorkers) { + this.minWorkers = Math.max(2, minWorkers); + this.maxWorkers = Math.max(this.minWorkers, maxWorkers); + } + } + + public record SchedulerConfig( + int totalMaxWorkers, + int queueCapacityPerPriority, + int keepAliveSeconds, + @NotNull Map perPrioritySizing + ) { + public SchedulerConfig { + if (queueCapacityPerPriority < 1) { + queueCapacityPerPriority = 1; + } + if (keepAliveSeconds < 1) { + keepAliveSeconds = 30; + } + } + } + + public enum RequestPriority { + UI_UX, + ASSEMBLY, + TILE, + TRACK, + EXPORT + } + + public enum CancellationDomain { + TILE, + TRACK, + EXPORT + } + + @FunctionalInterface + public interface ThrowingSupplier { + T get() throws Exception; + } + + private static final class RequestTaskCancelledException extends RuntimeException { + private RequestTaskCancelledException(final @NotNull String message) { + super(message); + } + } + + public RequestTaskScheduler(final @NotNull Vertx vertx, + final @NotNull SchedulerConfig config) { + this.vertx = vertx; + this.pools = new EnumMap<>(RequestPriority.class); + this.poolSizing = new EnumMap<>(RequestPriority.class); + this.activeWorkersByPriority = new EnumMap<>(RequestPriority.class); + this.generations = new EnumMap<>(CancellationDomain.class); + this.trackedTasks = new EnumMap<>(CancellationDomain.class); + + final var validatedSizing = new EnumMap(RequestPriority.class); + RequestPriority[] priorities = RequestPriority.values(); + int minTotal = 0; + for (final var priority : priorities) { + final var sizing = config.perPrioritySizing().getOrDefault(priority, new PoolSizing(2, 2)); + final var validated = new PoolSizing(sizing.getMinWorkers(), sizing.getMaxWorkers()); + validatedSizing.put(priority, validated); + this.poolSizing.put(priority, validated); + this.activeWorkersByPriority.put(priority, new AtomicInteger(0)); + minTotal += validated.getMinWorkers(); + } + final int totalMaxWorkers = Math.max(minTotal, config.totalMaxWorkers()); + final int elasticBudget = Math.max(0, totalMaxWorkers - minTotal); + this.globalElasticPermits = new Semaphore(elasticBudget, true); + + for (final var domain : CancellationDomain.values()) { + this.generations.put(domain, new AtomicLong(0L)); + this.trackedTasks.put(domain, new ConcurrentHashMap<>()); + } + + for (final var priority : priorities) { + final var sizing = validatedSizing.get(priority); + final var pool = new ThreadPoolExecutor( + sizing.getMinWorkers(), + sizing.getMaxWorkers(), + config.keepAliveSeconds(), + TimeUnit.SECONDS, + new java.util.concurrent.ArrayBlockingQueue<>(config.queueCapacityPerPriority()), + new NamedThreadFactory("hict-" + priority.name().toLowerCase() + "-worker"), + new ThreadPoolExecutor.AbortPolicy() + ); + pool.prestartAllCoreThreads(); + this.pools.put(priority, pool); + } + + log.info( + "Initialized request scheduler: totalMaxWorkers={}, reservedMinWorkers={}, elasticBudget={}, queueCapacityPerPriority={}, keepAliveSeconds={}, pools={}", + totalMaxWorkers, + minTotal, + elasticBudget, + config.queueCapacityPerPriority(), + config.keepAliveSeconds(), + validatedSizing + ); + } + + public void submit(final @NotNull RoutingContext ctx, + final @NotNull RequestPriority priority, + final @Nullable CancellationDomain domain, + final @NotNull ThrowingSupplier supplier, + final @NotNull Consumer onSuccess) { + submit(ctx, priority, domain, supplier, onSuccess, null); + } + + public void submit(final @NotNull RoutingContext ctx, + final @NotNull RequestPriority priority, + final @Nullable CancellationDomain domain, + final @NotNull ThrowingSupplier supplier, + final @NotNull Consumer onSuccess, + final @Nullable Runnable onCancelled) { + final var pool = this.pools.get(priority); + if (pool == null) { + dispatchFailure(ctx, new IllegalStateException("No pool is configured for priority " + priority)); + return; + } + final long generationSnapshot = domain != null ? this.currentGeneration(domain) : -1L; + final var selfRef = new java.util.concurrent.atomic.AtomicReference>(); + final FutureTask futureTask = new FutureTask<>(() -> { + final var activeWorkers = this.activeWorkersByPriority.get(priority); + final int minWorkers = this.poolSizing.get(priority).getMinWorkers(); + boolean elasticPermitAcquired = false; + boolean workerRegistered = false; + try { + this.ensureNotCancelled(domain, generationSnapshot); + final int workersNow = activeWorkers.incrementAndGet(); + workerRegistered = true; + if (workersNow > minWorkers) { + this.globalElasticPermits.acquire(); + elasticPermitAcquired = true; + } + this.ensureNotCancelled(domain, generationSnapshot); + final var value = supplier.get(); + this.ensureNotCancelled(domain, generationSnapshot); + dispatchSuccess(ctx, value, onSuccess); + } catch (final Throwable t) { + if (isCancellation(t)) { + dispatchCancelled(ctx, onCancelled); + } else { + dispatchFailure(ctx, t); + } + } finally { + if (elasticPermitAcquired) { + this.globalElasticPermits.release(); + } + if (workerRegistered) { + activeWorkers.decrementAndGet(); + } + final var task = selfRef.get(); + if (domain != null && task != null) { + unregisterTask(domain, generationSnapshot, task); + } + } + return null; + }); + selfRef.set(futureTask); + + if (domain != null) { + registerTask(domain, generationSnapshot, futureTask); + } + + try { + pool.execute(futureTask); + } catch (final java.util.concurrent.RejectedExecutionException rejection) { + if (domain != null) { + unregisterTask(domain, generationSnapshot, futureTask); + } + dispatchFailure(ctx, new IllegalStateException( + "Request queue is saturated for priority " + priority + ". Please retry.", rejection + )); + } + } + + public long bumpGeneration(final @NotNull CancellationDomain domain) { + final long newGeneration = this.generations.get(domain).incrementAndGet(); + final var mapForDomain = this.trackedTasks.get(domain); + mapForDomain.entrySet().removeIf(entry -> { + if (entry.getKey() >= newGeneration) { + return false; + } + entry.getValue().forEach(task -> task.cancel(true)); + return true; + }); + return newGeneration; + } + + public void bumpAssemblyGeneration() { + bumpGeneration(CancellationDomain.TILE); + bumpGeneration(CancellationDomain.TRACK); + bumpGeneration(CancellationDomain.EXPORT); + } + + public long currentGeneration(final @NotNull CancellationDomain domain) { + return this.generations.get(domain).get(); + } + + private void registerTask(final @NotNull CancellationDomain domain, + final long generation, + final @NotNull FutureTask task) { + this.trackedTasks + .get(domain) + .computeIfAbsent(generation, g -> ConcurrentHashMap.newKeySet()) + .add(task); + } + + private void unregisterTask(final @NotNull CancellationDomain domain, + final long generation, + final @NotNull FutureTask task) { + final var mapForDomain = this.trackedTasks.get(domain); + final var tasksForGeneration = mapForDomain.get(generation); + if (tasksForGeneration == null) { + return; + } + tasksForGeneration.remove(task); + if (tasksForGeneration.isEmpty()) { + mapForDomain.remove(generation, tasksForGeneration); + } + } + + private void ensureNotCancelled(final @Nullable CancellationDomain domain, + final long generationSnapshot) { + if (domain == null) { + return; + } + if (generationSnapshot != currentGeneration(domain)) { + throw new RequestTaskCancelledException( + "Request was cancelled for domain " + domain + " at generation " + generationSnapshot + ); + } + if (Thread.currentThread().isInterrupted()) { + throw new CancellationException("Worker thread interrupted"); + } + } + + private static boolean isCancellation(final @NotNull Throwable throwable) { + Throwable cursor = throwable; + while (cursor != null) { + if (cursor instanceof RequestTaskCancelledException + || cursor instanceof CancellationException + || cursor instanceof InterruptedException) { + return true; + } + cursor = cursor.getCause(); + } + return false; + } + + private void dispatchSuccess(final @NotNull RoutingContext ctx, + final @NotNull T value, + final @NotNull Consumer onSuccess) { + this.vertx.runOnContext(v -> { + if (ctx.response().ended()) { + return; + } + try { + onSuccess.accept(value); + } catch (final Throwable t) { + ctx.fail(t); + } + }); + } + + private void dispatchFailure(final @NotNull RoutingContext ctx, + final @NotNull Throwable throwable) { + this.vertx.runOnContext(v -> { + if (ctx.response().ended()) { + return; + } + ctx.fail(throwable); + }); + } + + private void dispatchCancelled(final @NotNull RoutingContext ctx, + final @Nullable Runnable onCancelled) { + this.vertx.runOnContext(v -> { + if (ctx.response().ended()) { + return; + } + if (onCancelled != null) { + onCancelled.run(); + } else { + ctx.response().setStatusCode(200).end(); + } + }); + } + + @Override + public void close() { + this.pools.values().forEach(pool -> { + pool.shutdownNow(); + try { + pool.awaitTermination(3, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + } + + private static final class NamedThreadFactory implements ThreadFactory { + private final @NotNull String prefix; + private final @NotNull AtomicInteger counter = new AtomicInteger(0); + + private NamedThreadFactory(final @NotNull String prefix) { + this.prefix = prefix; + } + + @Override + public @NotNull Thread newThread(final @NotNull Runnable r) { + final var thread = new Thread(r); + thread.setName(prefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + } +} diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/conversion/ConversionHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/conversion/ConversionHandlersHolder.java index 78e7383..869760b 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/conversion/ConversionHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/conversion/ConversionHandlersHolder.java @@ -2,7 +2,9 @@ import io.vertx.core.Vertx; import io.vertx.core.json.Json; +import io.vertx.core.shareddata.LocalMap; import io.vertx.ext.web.Router; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -10,6 +12,7 @@ import ru.itmo.ctlab.hict.hict_library.converters.HictToMcoolConverter; import ru.itmo.ctlab.hict.hict_library.converters.McoolToHictConverter; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.dto.response.conversion.ConversionJobDTO; import ru.itmo.ctlab.hict.hict_server.dto.response.conversion.ConversionSubmitResponseDTO; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; @@ -51,188 +54,330 @@ public class ConversionHandlersHolder extends HandlersHolder { @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/convert/upload").blockingHandler(ctx -> { - try { - cleanupOldJobs(); - - final var upload = ctx.fileUploads().stream().findFirst().orElseThrow(() -> new IllegalArgumentException("No file uploaded")); - final var sourcePath = Path.of(upload.uploadedFileName()); - - if (Files.size(sourcePath) > MAX_UPLOAD_BYTES) { - Files.deleteIfExists(sourcePath); - throw new IllegalArgumentException("Uploaded file is too large"); - } - final var req = ctx.request(); - final var direction = req.getParam("direction"); - final var outputExt = "hict-to-mcool".equals(direction) ? ".mcool" : ".hict.hdf5"; - final var outputPath = Files.createTempFile("hict-converter-out-", outputExt); - - final var resolutionCsv = req.getParam("resolutions"); - final var resolutions = parseResolutions(resolutionCsv); - final var compression = parseInteger(req.getParam("compression"), 6); - final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse(req.getParam("compressionAlgorithm") == null ? "deflate" : req.getParam("compressionAlgorithm")); - final var chunkSize = parseInteger(req.getParam("chunkSize"), 8192); - final var applyAgpRaw = Boolean.parseBoolean(req.getParam("applyAgp")); - final var agpPathRaw = req.getParam("agpPath") == null ? ConversionOptions.NO_AGP : req.getParam("agpPath"); - final var parallelism = parseInteger(req.getParam("parallelism"), Runtime.getRuntime().availableProcessors()); - - final var useAgp = "hict-to-mcool".equals(direction) && applyAgpRaw; - final var agpPath = useAgp ? agpPathRaw : ConversionOptions.NO_AGP; - final var options = new ConversionOptions(sourcePath, outputPath, resolutions, chunkSize, compression, compressionAlgorithm, agpPath, useAgp, parallelism); - - final var job = createJob(sourcePath, outputPath, direction, parallelism, true, true); - submitJob(job, options, ensureGroup("upload", 1)); - - ctx.response().end(Json.encode(new ConversionSubmitResponseDTO("submitted", job.jobId))); - } catch (IOException e) { - throw new RuntimeException(e); + router.post("/convert/upload").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + cleanupOldJobs(); + try { + final var upload = ctx.fileUploads().stream() + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No file uploaded")); + final var sourcePath = Path.of(upload.uploadedFileName()); + + if (Files.size(sourcePath) > MAX_UPLOAD_BYTES) { + Files.deleteIfExists(sourcePath); + throw new IllegalArgumentException("Uploaded file is too large"); + } + final var req = ctx.request(); + final var direction = req.getParam("direction"); + final var outputExt = "hict-to-mcool".equals(direction) ? ".mcool" : ".hict.hdf5"; + final var outputPath = Files.createTempFile("hict-converter-out-", outputExt); + + final var resolutionCsv = req.getParam("resolutions"); + final var resolutions = parseResolutions(resolutionCsv); + final var compression = parseInteger(req.getParam("compression"), 6); + final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse( + req.getParam("compressionAlgorithm") == null ? "deflate" : req.getParam("compressionAlgorithm") + ); + final var chunkSize = parseInteger(req.getParam("chunkSize"), 8192); + final var applyAgpRaw = Boolean.parseBoolean(req.getParam("applyAgp")); + final var agpPathRaw = req.getParam("agpPath") == null ? ConversionOptions.NO_AGP : req.getParam("agpPath"); + final var parallelism = parseInteger(req.getParam("parallelism"), Runtime.getRuntime().availableProcessors()); + + final var useAgp = "hict-to-mcool".equals(direction) && applyAgpRaw; + final var agpPath = useAgp ? agpPathRaw : ConversionOptions.NO_AGP; + final var options = new ConversionOptions( + sourcePath, + outputPath, + resolutions, + chunkSize, + compression, + compressionAlgorithm, + agpPath, + useAgp, + parallelism + ); + + final var job = createJob(sourcePath, outputPath, direction, parallelism, true, true); + submitJob(job, options, ensureGroup("upload", 1)); + return new ConversionSubmitResponseDTO("submitted", job.jobId); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + response -> ctx.response().end(Json.encode(response)), + () -> ctx.response().end(Json.encode(Map.of("status", "cancelled"))) + ); }); - router.post("/convert/jobs").blockingHandler(ctx -> { - cleanupOldJobs(); - final var requestJson = ctx.body().asJsonObject(); - final var filename = requestJson.getString("filename"); - if (filename == null || filename.isBlank()) { - throw new IllegalArgumentException("filename is required"); - } - final var direction = requestJson.getString("direction", "mcool-to-hict"); - final var parallelism = requestJson.getInteger("parallelism", Runtime.getRuntime().availableProcessors()); - final var resolutions = parseResolutions(requestJson.getString("resolutions")); - final var compression = requestJson.getInteger("compression", 6); - final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse(requestJson.getString("compressionAlgorithm", "deflate")); - final var chunkSize = requestJson.getInteger("chunkSize", 8192); - - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - throw new IllegalStateException("Data directory is not present in local map"); - } - final var dataDirectory = dataDirectoryWrapper.getPath(); - final var sourcePath = dataDirectory.resolve(filename).normalize(); - if (!sourcePath.startsWith(dataDirectory)) { - throw new IllegalArgumentException("Invalid filename"); - } - if (!Files.exists(sourcePath)) { - throw new IllegalArgumentException("Source file not found: " + filename); - } - - final var outputPath = deriveOutputPath(sourcePath); - if (Files.exists(outputPath)) { - throw new IllegalArgumentException("Output file already exists: " + outputPath.getFileName()); + router.post("/convert/jobs").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + cleanupOldJobs(); + final var requestJson = ctx.body().asJsonObject(); + final var filename = requestJson.getString("filename"); + if (filename == null || filename.isBlank()) { + throw new IllegalArgumentException("filename is required"); + } + final var direction = requestJson.getString("direction", "mcool-to-hict"); + final var parallelism = requestJson.getInteger("parallelism", Runtime.getRuntime().availableProcessors()); + final var resolutions = parseResolutions(requestJson.getString("resolutions")); + final var compression = requestJson.getInteger("compression", 6); + final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse(requestJson.getString("compressionAlgorithm", "deflate")); + final var chunkSize = requestJson.getInteger("chunkSize", 8192); + + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new IllegalStateException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); + final var sourcePath = dataDirectory.resolve(filename).normalize(); + if (!sourcePath.startsWith(dataDirectory)) { + throw new IllegalArgumentException("Invalid filename"); + } + if (!Files.exists(sourcePath)) { + throw new IllegalArgumentException("Source file not found: " + filename); + } - final var options = new ConversionOptions(sourcePath, outputPath, resolutions, chunkSize, compression, compressionAlgorithm, ConversionOptions.NO_AGP, false, parallelism); - final ConversionJob job; - try { - job = createJob(sourcePath, outputPath, direction, parallelism, false, false); - } catch (IOException e) { - throw new RuntimeException("Failed to create conversion job", e); - } - submitJob(job, options, ensureGroup(UUID.randomUUID().toString(), 1)); + final var outputPath = deriveOutputPath(sourcePath); + if (Files.exists(outputPath)) { + throw new IllegalArgumentException("Output file already exists: " + outputPath.getFileName()); + } - ctx.response().end(Json.encode(new ConversionSubmitResponseDTO("submitted", job.jobId))); + final var options = new ConversionOptions( + sourcePath, + outputPath, + resolutions, + chunkSize, + compression, + compressionAlgorithm, + ConversionOptions.NO_AGP, + false, + parallelism + ); + final ConversionJob job; + try { + job = createJob(sourcePath, outputPath, direction, parallelism, false, false); + } catch (IOException e) { + throw new RuntimeException("Failed to create conversion job", e); + } + submitJob(job, options, ensureGroup(UUID.randomUUID().toString(), 1)); + return new ConversionSubmitResponseDTO("submitted", job.jobId); + }, + response -> ctx.response().end(Json.encode(response)), + () -> ctx.response().end(Json.encode(Map.of("status", "cancelled"))) + ); }); - router.post("/convert/jobs/batch").blockingHandler(ctx -> { - cleanupOldJobs(); - final var requestJson = ctx.body().asJsonObject(); - final var files = requestJson.getJsonArray("files", null); - if (files == null || files.isEmpty()) { - throw new IllegalArgumentException("files is required"); - } - final var parallelJobs = Math.max(1, requestJson.getInteger("parallelJobs", 1)); - final var parallelism = requestJson.getInteger("parallelism", Runtime.getRuntime().availableProcessors()); - final var resolutions = parseResolutions(requestJson.getString("resolutions")); - final var compression = requestJson.getInteger("compression", 6); - final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse(requestJson.getString("compressionAlgorithm", "deflate")); - final var chunkSize = requestJson.getInteger("chunkSize", 8192); - - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - throw new IllegalStateException("Data directory is not present in local map"); - } - final var dataDirectory = dataDirectoryWrapper.getPath(); - - final var groupId = UUID.randomUUID().toString(); - final var group = ensureGroup(groupId, parallelJobs); - final var jobIds = new ArrayList(); - - for (int i = 0; i < files.size(); i++) { - final var filename = files.getString(i); - final var sourcePath = dataDirectory.resolve(filename).normalize(); - if (!sourcePath.startsWith(dataDirectory)) { - throw new IllegalArgumentException("Invalid filename: " + filename); - } - if (!Files.exists(sourcePath)) { - throw new IllegalArgumentException("Source file not found: " + filename); - } - final var outputPath = deriveOutputPath(sourcePath); - if (Files.exists(outputPath)) { - throw new IllegalArgumentException("Output file already exists: " + outputPath.getFileName()); - } - final var options = new ConversionOptions(sourcePath, outputPath, resolutions, chunkSize, compression, compressionAlgorithm, ConversionOptions.NO_AGP, false, parallelism); - final ConversionJob job; - try { - job = createJob(sourcePath, outputPath, "mcool-to-hict", parallelism, false, false); - } catch (IOException e) { - throw new RuntimeException("Failed to create conversion job for " + filename, e); - } - submitJob(job, options, group); - jobIds.add(job.jobId); + router.post("/convert/jobs/batch").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } - - ctx.response().end(Json.encode(Map.of("status", "submitted", "groupId", groupId, "jobIds", jobIds))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + cleanupOldJobs(); + final var requestJson = ctx.body().asJsonObject(); + final var files = requestJson.getJsonArray("files", null); + if (files == null || files.isEmpty()) { + throw new IllegalArgumentException("files is required"); + } + final var parallelJobs = Math.max(1, requestJson.getInteger("parallelJobs", 1)); + final var parallelism = requestJson.getInteger("parallelism", Runtime.getRuntime().availableProcessors()); + final var resolutions = parseResolutions(requestJson.getString("resolutions")); + final var compression = requestJson.getInteger("compression", 6); + final var compressionAlgorithm = ConversionOptions.CompressionAlgorithm.parse(requestJson.getString("compressionAlgorithm", "deflate")); + final var chunkSize = requestJson.getInteger("chunkSize", 8192); + + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new IllegalStateException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); + + final var groupId = UUID.randomUUID().toString(); + final var group = ensureGroup(groupId, parallelJobs); + final var jobIds = new ArrayList(); + + for (int i = 0; i < files.size(); i++) { + final var filename = files.getString(i); + final var sourcePath = dataDirectory.resolve(filename).normalize(); + if (!sourcePath.startsWith(dataDirectory)) { + throw new IllegalArgumentException("Invalid filename: " + filename); + } + if (!Files.exists(sourcePath)) { + throw new IllegalArgumentException("Source file not found: " + filename); + } + final var outputPath = deriveOutputPath(sourcePath); + if (Files.exists(outputPath)) { + throw new IllegalArgumentException("Output file already exists: " + outputPath.getFileName()); + } + final var options = new ConversionOptions( + sourcePath, + outputPath, + resolutions, + chunkSize, + compression, + compressionAlgorithm, + ConversionOptions.NO_AGP, + false, + parallelism + ); + final ConversionJob job; + try { + job = createJob(sourcePath, outputPath, "mcool-to-hict", parallelism, false, false); + } catch (IOException e) { + throw new RuntimeException("Failed to create conversion job for " + filename, e); + } + submitJob(job, options, group); + jobIds.add(job.jobId); + } + return Map.of("status", "submitted", "groupId", groupId, "jobIds", jobIds); + }, + response -> ctx.response().end(Json.encode(response)), + () -> ctx.response().end(Json.encode(Map.of("status", "cancelled"))) + ); }); - router.get("/convert/jobs").blockingHandler(ctx -> { - cleanupOldJobs(); - final var jobList = jobs.values().stream().map(ConversionJob::toDto).toList(); - ctx.response().end(Json.encode(jobList)); + router.get("/convert/jobs").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + cleanupOldJobs(); + return jobs.values().stream().map(ConversionJob::toDto).toList(); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); - router.post("/convert/jobs/list").blockingHandler(ctx -> { - cleanupOldJobs(); - final var jobList = jobs.values().stream().map(ConversionJob::toDto).toList(); - ctx.response().end(Json.encode(jobList)); + router.post("/convert/jobs/list").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + cleanupOldJobs(); + return jobs.values().stream().map(ConversionJob::toDto).toList(); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); - router.get("/convert/jobs/:jobId").blockingHandler(ctx -> { - final var job = jobs.get(ctx.pathParam("jobId")); - if (job == null) { - throw new IllegalArgumentException("Job not found"); + router.get("/convert/jobs/:jobId").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } - ctx.response().end(Json.encode(job.toDto())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var job = jobs.get(ctx.pathParam("jobId")); + if (job == null) { + throw new IllegalArgumentException("Job not found"); + } + return job.toDto(); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); - router.post("/convert/jobs/:jobId").blockingHandler(ctx -> { - final var job = jobs.get(ctx.pathParam("jobId")); - if (job == null) { - throw new IllegalArgumentException("Job not found"); + router.post("/convert/jobs/:jobId").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } - ctx.response().end(Json.encode(job.toDto())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var job = jobs.get(ctx.pathParam("jobId")); + if (job == null) { + throw new IllegalArgumentException("Job not found"); + } + return job.toDto(); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); - router.post("/convert/jobs/:jobId/stop").blockingHandler(ctx -> { - final var job = jobs.get(ctx.pathParam("jobId")); - if (job == null) { - throw new IllegalArgumentException("Job not found"); + router.post("/convert/jobs/:jobId/stop").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } - job.requestCancel(); - ctx.response().end(Json.encode(Map.of("status", "cancelling", "jobId", job.jobId))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var job = jobs.get(ctx.pathParam("jobId")); + if (job == null) { + throw new IllegalArgumentException("Job not found"); + } + job.requestCancel(); + return Map.of("status", "cancelling", "jobId", job.jobId); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); - router.get("/convert/download/:jobId").blockingHandler(ctx -> { - final var job = jobs.get(ctx.pathParam("jobId")); - if (job == null) { - throw new IllegalArgumentException("Job not found"); + router.get("/convert/download/:jobId").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; } - if (!"finished".equals(job.status)) { - throw new IllegalStateException("Job is not finished yet"); - } - if (!Files.exists(job.outputPath)) { - throw new IllegalStateException("Converted file was already cleaned up"); - } - ctx.response().putHeader("Content-Type", "application/octet-stream"); - ctx.response().putHeader("Content-Disposition", "attachment; filename=\"" + job.outputPath.getFileName() + "\""); - ctx.response().sendFile(job.outputPath.toString()); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + final var job = jobs.get(ctx.pathParam("jobId")); + if (job == null) { + throw new IllegalArgumentException("Job not found"); + } + if (!"finished".equals(job.status)) { + throw new IllegalStateException("Job is not finished yet"); + } + if (!Files.exists(job.outputPath)) { + throw new IllegalStateException("Converted file was already cleaned up"); + } + return job.outputPath; + }, + outputPath -> { + ctx.response().putHeader("Content-Type", "application/octet-stream"); + ctx.response().putHeader("Content-Disposition", "attachment; filename=\"" + outputPath.getFileName() + "\""); + ctx.response().sendFile(outputPath.toString()); + }, + () -> ctx.response().setStatusCode(200).end() + ); }); } @@ -390,6 +535,16 @@ private void cleanupOldJobs() { }); } + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } + private static class ConversionJobGroup { private final String groupId; private final ExecutorService executor; diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java index 45c8d2c..31170eb 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java @@ -38,6 +38,7 @@ import ru.itmo.ctlab.hict.hict_library.chunkedfile.Initializers; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.dto.response.assembly.AssemblyInfoDTO; import ru.itmo.ctlab.hict.hict_server.dto.response.fasta.FastaLinkResponseDTO; import ru.itmo.ctlab.hict.hict_server.dto.response.fileop.OpenFileResponseDTO; @@ -56,82 +57,94 @@ @Slf4j public class FileOpHandlersHolder extends HandlersHolder { private final Vertx vertx; + private record JsonRouteResult(int statusCode, @NotNull io.vertx.core.json.JsonObject payload) {} @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/open").blockingHandler(ctx -> { - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); + router.post("/open").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var dataDirectory = dataDirectoryWrapper.getPath(); - - final @NotNull var requestBody = ctx.body(); - final @NotNull var requestJSON = requestBody.asJsonObject(); - - final @Nullable var filename = requestJSON.getString("filename"); - final @Nullable var fastaFilename = requestJSON.getString("fastaFilename"); - - log.debug("Got filename: " + filename + " and FASTA filename: " + fastaFilename); - - if (filename == null) { - ctx.fail(new RuntimeException("Filename must be specified to open the file")); - return; - } - final boolean verbose = Boolean.parseBoolean(System.getProperty("HICT_VERBOSE", "false")); - if (verbose) { - log.info("Opening file " + filename); - } - - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var oldTrackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); - if (oldTrackManagerWrapper != null) { - oldTrackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); - oldTrackManagerWrapper.getTrack1DManager().close(); - } - map.remove("linkedFastaPath"); - map.remove("linkedFastaFilename"); - - final var progress = new io.vertx.core.json.JsonObject() - .put("stage", "starting") - .put("progress", 0.0); - map.put("openProgress", progress); - - final var chunkedFile = Initializers.withProgressReporter((stage, progressValue) -> { - map.put("openProgress", new io.vertx.core.json.JsonObject() - .put("stage", stage) - .put("progress", progressValue)); - if (verbose) { - log.info(String.format("Open progress: %s (%.1f%%)", stage, progressValue * 100.0)); - } - }, () -> new ChunkedFile( - new ChunkedFile.ChunkedFileOptions( - Path.of(dataDirectory.toString(), filename), - (int) map.getOrDefault("MIN_DS_POOL", 4), - (int) map.getOrDefault("MAX_DS_POOL", 16) - ) - )); - final var chunkedFileWrapper = new ShareableWrappers.ChunkedFileWrapper(chunkedFile); - - log.info("Putting chunkedFile into the local map"); - map.put("chunkedFile", chunkedFileWrapper); - map.put("openedFilename", filename); - - map.put("TileStatisticHolder", TileStatisticHolder.newDefaultStatisticHolder(chunkedFile.getResolutions().length)); - final var processedDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("processedDirectory"); - final var processedDirectory = processedDirectoryWrapper != null - ? processedDirectoryWrapper.getPath() - : dataDirectory.resolve("processed").normalize().toAbsolutePath(); - map.put("Track1DManager", new ShareableWrappers.Track1DManagerWrapper(new Track1DManager(dataDirectory, processedDirectory))); - - map.put("openProgress", new io.vertx.core.json.JsonObject() - .put("stage", "done") - .put("progress", 1.0)); - - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(generateOpenFileResponse(chunkedFile))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); + + final @NotNull var requestBody = ctx.body(); + final @NotNull var requestJSON = requestBody.asJsonObject(); + + final @Nullable var filename = requestJSON.getString("filename"); + final @Nullable var fastaFilename = requestJSON.getString("fastaFilename"); + + log.debug("Got filename: {} and FASTA filename: {}", filename, fastaFilename); + if (filename == null) { + throw new RuntimeException("Filename must be specified to open the file"); + } + final boolean verbose = Boolean.parseBoolean(System.getProperty("HICT_VERBOSE", "false")); + if (verbose) { + log.info("Opening file {}", filename); + } + + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var oldTrackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (oldTrackManagerWrapper != null) { + oldTrackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); + oldTrackManagerWrapper.getTrack1DManager().close(); + } + map.remove("linkedFastaPath"); + map.remove("linkedFastaFilename"); + + map.put("openProgress", new io.vertx.core.json.JsonObject() + .put("stage", "starting") + .put("progress", 0.0)); + + final var chunkedFile = Initializers.withProgressReporter((stage, progressValue) -> { + map.put("openProgress", new io.vertx.core.json.JsonObject() + .put("stage", stage) + .put("progress", progressValue)); + if (verbose) { + log.info(String.format("Open progress: %s (%.1f%%)", stage, progressValue * 100.0)); + } + }, () -> new ChunkedFile( + new ChunkedFile.ChunkedFileOptions( + Path.of(dataDirectory.toString(), filename), + (int) map.getOrDefault("MIN_DS_POOL", 4), + (int) map.getOrDefault("MAX_DS_POOL", 16) + ) + )); + final var chunkedFileWrapper = new ShareableWrappers.ChunkedFileWrapper(chunkedFile); + log.info("Putting chunkedFile into the local map"); + map.put("chunkedFile", chunkedFileWrapper); + map.put("openedFilename", filename); + map.put("TileStatisticHolder", TileStatisticHolder.newDefaultStatisticHolder(chunkedFile.getResolutions().length)); + + final var processedDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("processedDirectory"); + final var processedDirectory = processedDirectoryWrapper != null + ? processedDirectoryWrapper.getPath() + : dataDirectory.resolve("processed").normalize().toAbsolutePath(); + map.put("Track1DManager", new ShareableWrappers.Track1DManagerWrapper(new Track1DManager(dataDirectory, processedDirectory))); + + final var schedulerWrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (schedulerWrapper != null) { + schedulerWrapper.getRequestTaskScheduler().bumpAssemblyGeneration(); + } + + map.put("openProgress", new io.vertx.core.json.JsonObject() + .put("stage", "done") + .put("progress", 1.0)); + return generateOpenFileResponse(chunkedFile); + }, + response -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(response)) + ); }); router.post("/open_progress").handler(ctx -> { @@ -150,130 +163,171 @@ public void addHandlersToRouter(final @NotNull Router router) { .end(((io.vertx.core.json.JsonObject) progressObj).encode()); }); - router.post("/attach").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.response() - .setStatusCode(404) - .putHeader("content-type", "application/json") - .end(Json.encode(new io.vertx.core.json.JsonObject().put("error", "No session to attach"))); + router.post("/attach").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - final var filename = (String) map.getOrDefault("openedFilename", ""); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode( - new io.vertx.core.json.JsonObject() - .put("filename", filename) - .put("fastaFilename", map.getOrDefault("linkedFastaFilename", "")) - .put("openFileResponse", generateOpenFileResponse(chunkedFile)) - )); - }); - - router.post("/close").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper != null) { - try { - chunkedFileWrapper.getChunkedFile().close(); - } catch (Exception e) { - log.warn("Failed to close chunked file", e); - } - } - map.remove("chunkedFile"); - map.remove("TileStatisticHolder"); - map.remove("openedFilename"); - map.remove("linkedFastaPath"); - map.remove("linkedFastaFilename"); - final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.remove("Track1DManager"); - if (trackManagerWrapper != null) { - trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); - trackManagerWrapper.getTrack1DManager().close(); - } - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(new io.vertx.core.json.JsonObject().put("status", "closed"))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + return new JsonRouteResult( + 404, + new io.vertx.core.json.JsonObject().put("error", "No session to attach") + ); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + final var filename = (String) map.getOrDefault("openedFilename", ""); + return new JsonRouteResult( + 200, + new io.vertx.core.json.JsonObject() + .put("filename", filename) + .put("fastaFilename", map.getOrDefault("linkedFastaFilename", "")) + .put("openFileResponse", generateOpenFileResponse(chunkedFile)) + ); + }, + response -> ctx.response() + .setStatusCode(response.statusCode()) + .putHeader("content-type", "application/json") + .end(response.payload().encode()) + ); }); - router.post("/link_fasta").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new IllegalStateException("Open a Hi-C file before linking FASTA")); - return; - } - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); - return; - } - final var requestJSON = ctx.body().asJsonObject(); - final var fastaFilename = requestJSON.getString("fastaFilename"); - final boolean allowMismatch = requestJSON.getBoolean("allowMismatch", false); - if (fastaFilename == null || fastaFilename.isBlank()) { - ctx.fail(new IllegalArgumentException("FASTA filename is required")); + router.post("/close").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper != null) { + try { + chunkedFileWrapper.getChunkedFile().close(); + } catch (Exception e) { + log.warn("Failed to close chunked file", e); + } + } + map.remove("chunkedFile"); + map.remove("TileStatisticHolder"); + map.remove("openedFilename"); + map.remove("linkedFastaPath"); + map.remove("linkedFastaFilename"); + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.remove("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource(java.util.Map.of()); + trackManagerWrapper.getTrack1DManager().close(); + } + final var schedulerWrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (schedulerWrapper != null) { + schedulerWrapper.getRequestTaskScheduler().bumpAssemblyGeneration(); + } + return new io.vertx.core.json.JsonObject().put("status", "closed"); + }, + response -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(response)) + ); + }); - final Path fastaPath = dataDirectoryWrapper.getPath().resolve(fastaFilename).normalize().toAbsolutePath(); - if (!fastaPath.startsWith(dataDirectoryWrapper.getPath())) { - ctx.fail(new IllegalArgumentException("FASTA path " + fastaFilename + " is outside DATA_DIR")); - return; - } - if (!Files.exists(fastaPath) || !Files.isRegularFile(fastaPath)) { - ctx.fail(new IllegalArgumentException("FASTA file " + fastaFilename + " does not exist")); + router.post("/link_fasta").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - - final var report = chunkedFileWrapper.getChunkedFile().getFastaProcessor().analyzeLinkCandidate(fastaPath); - final boolean requiresConfirmation = report.hasWarnings() && !allowMismatch; - if (!requiresConfirmation) { - map.put("linkedFastaPath", new ShareableWrappers.PathWrapper(fastaPath)); - map.put("linkedFastaFilename", fastaFilename); - final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); - if (trackManagerWrapper != null) { - trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource( - chunkedFileWrapper.getChunkedFile().getFastaProcessor().buildSourceNameAliases(fastaPath) - ); - } - } - - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(FastaLinkResponseDTO.fromReport(report, !requiresConfirmation, requiresConfirmation))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new IllegalStateException("Open a Hi-C file before linking FASTA"); + } + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) map.get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var requestJSON = ctx.body().asJsonObject(); + final var fastaFilename = requestJSON.getString("fastaFilename"); + final boolean allowMismatch = requestJSON.getBoolean("allowMismatch", false); + if (fastaFilename == null || fastaFilename.isBlank()) { + throw new IllegalArgumentException("FASTA filename is required"); + } + + final Path fastaPath = dataDirectoryWrapper.getPath().resolve(fastaFilename).normalize().toAbsolutePath(); + if (!fastaPath.startsWith(dataDirectoryWrapper.getPath())) { + throw new IllegalArgumentException("FASTA path " + fastaFilename + " is outside DATA_DIR"); + } + if (!Files.exists(fastaPath) || !Files.isRegularFile(fastaPath)) { + throw new IllegalArgumentException("FASTA file " + fastaFilename + " does not exist"); + } + + final var report = chunkedFileWrapper.getChunkedFile().getFastaProcessor().analyzeLinkCandidate(fastaPath); + final boolean requiresConfirmation = report.hasWarnings() && !allowMismatch; + if (!requiresConfirmation) { + map.put("linkedFastaPath", new ShareableWrappers.PathWrapper(fastaPath)); + map.put("linkedFastaFilename", fastaFilename); + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().setLinkedFastaAliasesBySource( + chunkedFileWrapper.getChunkedFile().getFastaProcessor().buildSourceNameAliases(fastaPath) + ); + } + } + return FastaLinkResponseDTO.fromReport(report, !requiresConfirmation, requiresConfirmation); + }, + response -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(response)) + ); }); - router.post("/get_fasta_for_assembly").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new IllegalStateException("Open a Hi-C file before exporting FASTA")); - return; - } - final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); - if (fastaPathWrapper == null) { - ctx.fail(new IllegalStateException("Link a FASTA file before exporting FASTA")); + router.post("/get_fasta_for_assembly").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var fasta = chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportAssembly(fastaPathWrapper.getPath()); - ctx.response() - .setChunked(true) - .putHeader("Content-Type", "text/plain") - .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new IllegalStateException("Open a Hi-C file before exporting FASTA"); + } + final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); + if (fastaPathWrapper == null) { + throw new IllegalStateException("Link a FASTA file before exporting FASTA"); + } + return chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportAssembly(fastaPathWrapper.getPath()); + }, + fasta -> ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())), + () -> ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer("", StandardCharsets.UTF_8.name())) + ); }); - router.post("/get_fasta_for_selection").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new IllegalStateException("Open a Hi-C file before exporting FASTA")); - return; - } - final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); - if (fastaPathWrapper == null) { - ctx.fail(new IllegalStateException("Link a FASTA file before exporting FASTA")); + router.post("/get_fasta_for_selection").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } final var requestJSON = ctx.body().asJsonObject(); @@ -285,76 +339,123 @@ public void addHandlersToRouter(final @NotNull Router router) { ctx.fail(new IllegalArgumentException("Selection coordinates must be provided")); return; } - final var fasta = chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportSelection( - fastaPathWrapper.getPath(), - fromBpX, - fromBpY, - toBpX, - toBpY + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new IllegalStateException("Open a Hi-C file before exporting FASTA"); + } + final var fastaPathWrapper = (ShareableWrappers.PathWrapper) map.get("linkedFastaPath"); + if (fastaPathWrapper == null) { + throw new IllegalStateException("Link a FASTA file before exporting FASTA"); + } + return chunkedFileWrapper.getChunkedFile().getFastaProcessor().exportSelection( + fastaPathWrapper.getPath(), + fromBpX, + fromBpY, + toBpX, + toBpY + ); + }, + fasta -> ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())), + () -> ctx.response() + .setChunked(true) + .putHeader("Content-Type", "text/plain") + .end(Buffer.buffer("", StandardCharsets.UTF_8.name())) ); - ctx.response() - .setChunked(true) - .putHeader("Content-Type", "text/plain") - .end(Buffer.buffer(fasta, StandardCharsets.UTF_8.name())); }); - router.post("/get_agp_for_assembly").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); + router.post("/get_agp_for_assembly").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - log.debug("Got ChunkedFile from map"); - final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); - final long defaultSpacerLength = requestJSON.getLong("defaultSpacerLength", 1000L); - - final var buffer = Buffer.buffer(); - - chunkedFile.getAgpProcessor().getAGPStream(defaultSpacerLength).sequential().forEach(s -> buffer.appendBytes(s.getBytes(StandardCharsets.UTF_8))); - - ctx.response().setChunked(true).putHeader("Content-Type", "text/plain").end(buffer); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + + final var buffer = Buffer.buffer(); + chunkedFile.getAgpProcessor().getAGPStream(defaultSpacerLength).sequential().forEach(s -> buffer.appendBytes(s.getBytes(StandardCharsets.UTF_8))); + return buffer; + }, + buffer -> ctx.response().setChunked(true).putHeader("Content-Type", "text/plain").end(buffer), + () -> ctx.response().setChunked(true).putHeader("Content-Type", "text/plain").end(Buffer.buffer()) + ); }); - router.post("/load_agp").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); + router.post("/load_agp").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - log.debug("Got ChunkedFile from map"); - - final @NotNull var requestBody = ctx.body(); - final @NotNull var requestJSON = requestBody.asJsonObject(); - - final var agpFilename = Objects.requireNonNull(requestJSON.getString("agpFilename"), "AGP filename must be provided to load it."); - - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); - return; - } - final var dataDirectory = dataDirectoryWrapper.getPath(); - - final var agpFile = Path.of(dataDirectory.toString(), agpFilename); - try (final var reader = Files.newBufferedReader(agpFile, StandardCharsets.UTF_8)) { - chunkedFile.importAGP(reader); - } catch (IOException | NoSuchFieldException e) { - throw new RuntimeException(e); - } - - ctx.response().end(Json.encode(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + log.debug("Got ChunkedFile from map"); + + final @NotNull var requestBody = ctx.body(); + final @NotNull var requestJSON = requestBody.asJsonObject(); + final var agpFilename = Objects.requireNonNull(requestJSON.getString("agpFilename"), "AGP filename must be provided to load it."); + + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); + final var agpFile = Path.of(dataDirectory.toString(), agpFilename); + try (final var reader = Files.newBufferedReader(agpFile, StandardCharsets.UTF_8)) { + chunkedFile.importAGP(reader); + } catch (IOException | NoSuchFieldException e) { + throw new RuntimeException(e); + } + final var schedulerWrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (schedulerWrapper != null) { + schedulerWrapper.getRequestTaskScheduler().bumpAssemblyGeneration(); + } + return AssemblyInfoDTO.generateFromChunkedFile(chunkedFile); + }, + response -> ctx.response().end(Json.encode(response)) + ); }); } + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } + private @NotNull OpenFileResponseDTO generateOpenFileResponse(final @NotNull ChunkedFile chunkedFile) { final var resolutionsWithoutZero = Arrays.stream(chunkedFile.getResolutions()).skip(1L).toArray(); ArrayUtils.reverse(resolutionsWithoutZero); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java index fbfbcbb..95f88ab 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/files/FSHandlersHolder.java @@ -26,11 +26,13 @@ import io.vertx.core.Vertx; import io.vertx.core.json.Json; +import io.vertx.core.shareddata.LocalMap; import io.vertx.ext.web.Router; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; import java.io.IOException; @@ -49,80 +51,126 @@ public class FSHandlersHolder extends HandlersHolder { @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/list_files").blockingHandler(ctx -> { - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); + router.post("/list_files").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var dataDirectory = dataDirectoryWrapper.getPath(); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); - final List files; - try (final var fileStream = Files.walk(dataDirectory)) { - files = fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).collect(Collectors.toList()); - } catch (final IOException e) { - throw new RuntimeException(e); - } - ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)); + try (final var fileStream = Files.walk(dataDirectory)) { + return fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).collect(Collectors.toList()); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }, + files -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)) + ); }); - router.post("/list_agp_files").blockingHandler(ctx -> { - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); + router.post("/list_agp_files").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var dataDirectory = dataDirectoryWrapper.getPath(); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); - final List files; - try (final var fileStream = Files.walk(dataDirectory)) { - files = fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).filter(p -> p.toLowerCase().endsWith(".agp")).collect(Collectors.toList()); - } catch (final IOException e) { - throw new RuntimeException(e); - } - ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)); + try (final var fileStream = Files.walk(dataDirectory)) { + return fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).filter(p -> p.toLowerCase().endsWith(".agp")).collect(Collectors.toList()); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }, + files -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)) + ); }); - router.post("/list_fasta_files").blockingHandler(ctx -> { - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); + router.post("/list_fasta_files").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var dataDirectory = dataDirectoryWrapper.getPath(); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); - final List files; - try (final var fileStream = Files.walk(dataDirectory)) { - files = fileStream - .filter(Files::isRegularFile) - .map(dataDirectory::relativize) - .map(Object::toString) - .filter(FSHandlersHolder::isFastaFilename) - .collect(Collectors.toList()); - } catch (final IOException e) { - throw new RuntimeException(e); - } - ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)); + try (final var fileStream = Files.walk(dataDirectory)) { + return fileStream + .filter(Files::isRegularFile) + .map(dataDirectory::relativize) + .map(Object::toString) + .filter(FSHandlersHolder::isFastaFilename) + .collect(Collectors.toList()); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }, + files -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)) + ); }); - router.post("/list_coolers").blockingHandler(ctx -> { - final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); - if (dataDirectoryWrapper == null) { - ctx.fail(new RuntimeException("Data directory is not present in local map")); + router.post("/list_coolers").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var dataDirectory = dataDirectoryWrapper.getPath(); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final var dataDirectoryWrapper = (ShareableWrappers.PathWrapper) vertx.sharedData().getLocalMap("hict_server").get("dataDirectory"); + if (dataDirectoryWrapper == null) { + throw new RuntimeException("Data directory is not present in local map"); + } + final var dataDirectory = dataDirectoryWrapper.getPath(); - final List files; - try (final var fileStream = Files.walk(dataDirectory)) { - files = fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).filter(p -> p.toLowerCase().endsWith(".cool") || p.toLowerCase().endsWith(".mcool")).collect(Collectors.toList()); - } catch (final IOException e) { - throw new RuntimeException(e); - } - ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)); + try (final var fileStream = Files.walk(dataDirectory)) { + return fileStream.filter(Files::isRegularFile).map(dataDirectory::relativize).map(Object::toString).filter(p -> p.toLowerCase().endsWith(".cool") || p.toLowerCase().endsWith(".mcool")).collect(Collectors.toList()); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }, + files -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)) + ); }); } + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } + private static boolean isFastaFilename(final @NotNull String path) { final var lowered = path.toLowerCase(); return FASTA_SUFFIXES.stream().anyMatch(lowered::endsWith); diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java index 06526e0..cd710bd 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java @@ -33,6 +33,7 @@ import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.dto.request.names.ImportNameMappingRequestDTO; import ru.itmo.ctlab.hict.hict_server.dto.request.names.RenameContigRequestDTO; import ru.itmo.ctlab.hict.hict_server.dto.request.names.RenameScaffoldRequestDTO; @@ -53,95 +54,125 @@ public class NameMappingHandlersHolder extends HandlersHolder { @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/names/contig").blockingHandler(ctx -> { - final var request = RenameContigRequestDTO.fromJSONObject(ctx.body().asJsonObject()); - final var chunkedFile = extractChunkedFile(ctx); - if (chunkedFile == null) { + router.post("/names/contig").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - - final var newName = normalizeName(request.newName()); - validateContigRename(chunkedFile, request.contigId(), newName); - chunkedFile.setContigNameOverride(request.contigId(), newName); - - final var newVersion = incrementVersionAndResetTileStats(chunkedFile); - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); + final var request = RenameContigRequestDTO.fromJSONObject(ctx.body().asJsonObject()); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final var chunkedFile = extractChunkedFile(); + final var newName = normalizeName(request.newName()); + validateContigRename(chunkedFile, request.contigId(), newName); + chunkedFile.setContigNameOverride(request.contigId(), newName); + final var newVersion = incrementVersionAndResetTileStats(chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); }); - router.post("/names/scaffold").blockingHandler(ctx -> { - final var request = RenameScaffoldRequestDTO.fromJSONObject(ctx.body().asJsonObject()); - final var chunkedFile = extractChunkedFile(ctx); - if (chunkedFile == null) { + router.post("/names/scaffold").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - - final var newName = normalizeName(request.newName()); - validateScaffoldRename(chunkedFile, request.scaffoldId(), newName); - chunkedFile.setScaffoldNameOverride(request.scaffoldId(), newName); - - final var newVersion = incrementVersionAndResetTileStats(chunkedFile); - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); + final var request = RenameScaffoldRequestDTO.fromJSONObject(ctx.body().asJsonObject()); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final var chunkedFile = extractChunkedFile(); + final var newName = normalizeName(request.newName()); + validateScaffoldRename(chunkedFile, request.scaffoldId(), newName); + chunkedFile.setScaffoldNameOverride(request.scaffoldId(), newName); + final var newVersion = incrementVersionAndResetTileStats(chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); }); - router.get("/names/export").blockingHandler(ctx -> { - final var chunkedFile = extractChunkedFile(ctx); - if (chunkedFile == null) { + router.get("/names/export").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - - final var contigs = chunkedFile.getContigTree().getOrderedContigList().stream().map(tuple -> - new NameMappingDTO.ContigNameMappingDTO( - tuple.descriptor().getContigId(), - chunkedFile.getContigOriginalName(tuple.descriptor().getContigId()), - chunkedFile.getContigDisplayName(tuple.descriptor().getContigId()) - ) - ).toList(); - - final var scaffolds = chunkedFile.getScaffoldTree().getScaffoldList().stream().map(tuple -> - new NameMappingDTO.ScaffoldNameMappingDTO( - tuple.scaffoldDescriptor().scaffoldId(), - chunkedFile.getScaffoldOriginalName(tuple.scaffoldDescriptor().scaffoldId()), - chunkedFile.getScaffoldDisplayName(tuple.scaffoldDescriptor().scaffoldId()) - ) - ).toList(); - - ctx.response().end(Json.encode(new NameMappingDTO(contigs, scaffolds))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.EXPORT, + RequestTaskScheduler.CancellationDomain.EXPORT, + () -> { + final var chunkedFile = extractChunkedFile(); + final var contigs = chunkedFile.getContigTree().getOrderedContigList().stream().map(tuple -> + new NameMappingDTO.ContigNameMappingDTO( + tuple.descriptor().getContigId(), + chunkedFile.getContigOriginalName(tuple.descriptor().getContigId()), + chunkedFile.getContigDisplayName(tuple.descriptor().getContigId()) + ) + ).toList(); + final var scaffolds = chunkedFile.getScaffoldTree().getScaffoldList().stream().map(tuple -> + new NameMappingDTO.ScaffoldNameMappingDTO( + tuple.scaffoldDescriptor().scaffoldId(), + chunkedFile.getScaffoldOriginalName(tuple.scaffoldDescriptor().scaffoldId()), + chunkedFile.getScaffoldDisplayName(tuple.scaffoldDescriptor().scaffoldId()) + ) + ).toList(); + return new NameMappingDTO(contigs, scaffolds); + }, + dto -> ctx.response().end(Json.encode(dto)), + () -> ctx.response().end(Json.encode(new NameMappingDTO(java.util.List.of(), java.util.List.of()))) + ); }); - router.post("/names/import").blockingHandler(ctx -> { - final var request = ImportNameMappingRequestDTO.fromJSONObject(ctx.body().asJsonObject()); - final var chunkedFile = extractChunkedFile(ctx); - if (chunkedFile == null) { + router.post("/names/import").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - - final Map contigUpdates = new HashMap<>(); - request.contigs().forEach(entry -> contigUpdates.put(entry.contigId(), normalizeName(entry.name()))); - final Map scaffoldUpdates = new HashMap<>(); - request.scaffolds().forEach(entry -> scaffoldUpdates.put(entry.scaffoldId(), normalizeName(entry.name()))); - - validateContigMappingImport(chunkedFile, contigUpdates); - validateScaffoldMappingImport(chunkedFile, scaffoldUpdates); - - contigUpdates.forEach(chunkedFile::setContigNameOverride); - scaffoldUpdates.forEach(chunkedFile::setScaffoldNameOverride); - - final var newVersion = incrementVersionAndResetTileStats(chunkedFile); - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); + final var request = ImportNameMappingRequestDTO.fromJSONObject(ctx.body().asJsonObject()); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final var chunkedFile = extractChunkedFile(); + + final Map contigUpdates = new HashMap<>(); + request.contigs().forEach(entry -> contigUpdates.put(entry.contigId(), normalizeName(entry.name()))); + final Map scaffoldUpdates = new HashMap<>(); + request.scaffolds().forEach(entry -> scaffoldUpdates.put(entry.scaffoldId(), normalizeName(entry.name()))); + + validateContigMappingImport(chunkedFile, contigUpdates); + validateScaffoldMappingImport(chunkedFile, scaffoldUpdates); + + contigUpdates.forEach(chunkedFile::setContigNameOverride); + scaffoldUpdates.forEach(chunkedFile::setScaffoldNameOverride); + + final var newVersion = incrementVersionAndResetTileStats(chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); }); } - private ChunkedFile extractChunkedFile(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + private ChunkedFile extractChunkedFile() { final @NotNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); - return null; + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); } return chunkedFileWrapper.getChunkedFile(); } - private long incrementVersionAndResetTileStats(final @NotNull ChunkedFile chunkedFile) { + private long incrementVersionAndResetTileStats(final @NotNull ChunkedFile chunkedFile, + final @NotNull RequestTaskScheduler scheduler) { final @NotNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); if (stats == null) { @@ -149,9 +180,20 @@ private long incrementVersionAndResetTileStats(final @NotNull ChunkedFile chunke } final var newStats = TileStatisticHolder.resetRangesWithIncrementedVersion(stats, chunkedFile.getResolutions().length); map.put("TileStatisticHolder", newStats); + scheduler.bumpAssemblyGeneration(); return newStats.versionCounter().get(); } + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } + private static @NotNull String normalizeName(final String name) { if (name == null) { return ""; diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java index 1ba8e9b..2875f05 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java @@ -36,6 +36,7 @@ import ru.itmo.ctlab.hict.hict_library.chunkedfile.resolution.ResolutionDescriptor; import ru.itmo.ctlab.hict.hict_library.domain.QueryLengthUnit; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.dto.request.scaffolding.*; import ru.itmo.ctlab.hict.hict_server.dto.response.assembly.AssemblyInfoDTO; import ru.itmo.ctlab.hict.hict_server.dto.response.assembly.AssemblyInfoWithVersionDTO; @@ -49,159 +50,198 @@ public class ScaffoldingOpHandlersHolder extends HandlersHolder { @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/reverse_selection_range").blockingHandler(ctx -> { + router.post("/reverse_selection_range").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = ReverseSelectionRangeRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().reverseSelectionRangeBp(request.startBP(), request.endBP()); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().reverseSelectionRangeBp(request.startBP(), request.endBP()); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); + }); + router.post("/move_selection_range").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); - }); - router.post("/move_selection_range").blockingHandler(ctx -> { final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = MoveSelectionRangeRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().moveSelectionRangeBp(request.startBP(), request.endBP(), request.targetStartBP()); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().moveSelectionRangeBp(request.startBP(), request.endBP(), request.targetStartBP()); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); + }); + router.post("/split_contig_at_bin").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); - }); - router.post("/split_contig_at_bin").blockingHandler(ctx -> { final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = SplitContigRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().splitContigAtBin(request.splitPx(), ResolutionDescriptor.fromBpResolution(request.bpResolution(), chunkedFile), QueryLengthUnit.PIXELS); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().splitContigAtBin(request.splitPx(), ResolutionDescriptor.fromBpResolution(request.bpResolution(), chunkedFile), QueryLengthUnit.PIXELS); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); + }); + router.post("/group_contigs_into_scaffold").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); - }); - router.post("/group_contigs_into_scaffold").blockingHandler(ctx -> { final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = ScaffoldRegionRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().scaffoldRegion(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS, null); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().scaffoldRegion(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS, null); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); + }); + router.post("/ungroup_contigs_from_scaffold").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); - }); - router.post("/ungroup_contigs_from_scaffold").blockingHandler(ctx -> { final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = UnscaffoldRegionRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().unscaffoldRegion(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().unscaffoldRegion(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); + }); + router.post("/move_selection_to_debris").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); - }); - router.post("/move_selection_to_debris").blockingHandler(ctx -> { final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = MoveSelectionToDebrisRequestDTO.fromJSONObject(requestJSON); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFile = extractChunkedFile(map, ctx); - if (chunkedFile == null) { - return; - } - log.debug("Got ChunkedFile from map"); - - chunkedFile.scaffoldingOperations().moveRegionToDebris(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS); - - final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, ctx); - if (newVersion == null) { - return; - } - ctx.response().end(Json.encode(new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFile = extractChunkedFile(map); + log.debug("Got ChunkedFile from map"); + + chunkedFile.scaffoldingOperations().moveRegionToDebris(request.startBP(), request.endBP(), ResolutionDescriptor.fromResolutionOrder(0), QueryLengthUnit.BASE_PAIRS); + final var newVersion = incrementVersionAndResetTileStats(map, chunkedFile, scheduler); + return new AssemblyInfoWithVersionDTO(AssemblyInfoDTO.generateFromChunkedFile(chunkedFile), newVersion); + }, + dto -> ctx.response().end(Json.encode(dto)) + ); }); } - private ChunkedFile extractChunkedFile(final @NotNull LocalMap map, final @NotNull io.vertx.ext.web.RoutingContext ctx) { + private ChunkedFile extractChunkedFile(final @NotNull LocalMap map) { final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); - return null; + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); } return chunkedFileWrapper.getChunkedFile(); } - private Long incrementVersionAndResetTileStats(final @NotNull LocalMap map, + private long incrementVersionAndResetTileStats(final @NotNull LocalMap map, final @NotNull ChunkedFile chunkedFile, - final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull RequestTaskScheduler scheduler) { final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); if (stats == null) { - ctx.fail(new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?")); - return null; + throw new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?"); } final var newStats = TileStatisticHolder.resetRangesWithIncrementedVersion(stats, chunkedFile.getResolutions().length); map.put("TileStatisticHolder", newStats); + scheduler.bumpAssemblyGeneration(); return newStats.versionCounter().get(); } + + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tiles/TileHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tiles/TileHandlersHolder.java index d3b9511..cfe2507 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tiles/TileHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tiles/TileHandlersHolder.java @@ -35,6 +35,7 @@ import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_library.chunkedfile.resolution.ResolutionDescriptor; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.dto.symmetric.visualization.VisualizationOptionsDTO; import ru.itmo.ctlab.hict.hict_server.handlers.util.TileStatisticHolder; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; @@ -43,6 +44,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Base64; +import java.util.Collections; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -51,224 +53,268 @@ @Slf4j public class TileHandlersHolder extends HandlersHolder { private final Vertx vertx; + private static final String TRANSPARENT_PNG_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO7Z3ioAAAAASUVORK5CYII="; + private static final byte[] TRANSPARENT_PNG_BYTES = Base64.getDecoder().decode(TRANSPARENT_PNG_BASE64); @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/set_visualization_options").blockingHandler(ctx -> { + router.post("/set_visualization_options").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final @NotNull var requestBody = ctx.body(); final @NotNull var requestJSON = requestBody.asJsonObject(); final @NotNull @NonNull var request = VisualizationOptionsDTO.fromJSONObject(requestJSON); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + map.put("visualizationOptions", new ShareableWrappers.SimpleVisualizationOptionsWrapper(request.toEntity())); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + log.debug("Got ChunkedFile from map"); - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - map.put("visualizationOptions", new ShareableWrappers.SimpleVisualizationOptionsWrapper(request.toEntity())); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); - return; - } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - log.debug("Got ChunkedFile from map"); - - final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); - if (stats == null) { - ctx.fail(new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?")); - return; - } - map.put("TileStatisticHolder", TileStatisticHolder.resetRangesKeepingVersion(stats, chunkedFile.getResolutions().length)); - final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); - if (visualizationOptionsWrapper == null) { - ctx.fail(new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?")); - return; - } - final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); - ctx.response() - .putHeader("content-type", "application/json") - .setStatusCode(200) - .end(Json.encode(VisualizationOptionsDTO.fromEntity(options, chunkedFile))); + final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); + if (stats == null) { + throw new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?"); + } + map.put("TileStatisticHolder", TileStatisticHolder.resetRangesKeepingVersion(stats, chunkedFile.getResolutions().length)); + final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); + if (visualizationOptionsWrapper == null) { + throw new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?"); + } + final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); + return VisualizationOptionsDTO.fromEntity(options, chunkedFile); + }, + responseDto -> ctx.response() + .putHeader("content-type", "application/json") + .setStatusCode(200) + .end(Json.encode(responseDto)) + ); }); - router.post("/get_visualization_options").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - log.debug("Got map"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); + router.post("/get_visualization_options").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - log.debug("Got ChunkedFile from map"); - final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); - if (visualizationOptionsWrapper == null) { - ctx.fail(new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?")); - return; - } - final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); - ctx.response() - .putHeader("content-type", "application/json") - .setStatusCode(200) - .end(Json.encode(VisualizationOptionsDTO.fromEntity(options, chunkedFile))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + log.debug("Got map"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + log.debug("Got ChunkedFile from map"); + final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); + if (visualizationOptionsWrapper == null) { + throw new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?"); + } + final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); + return VisualizationOptionsDTO.fromEntity(options, chunkedFile); + }, + responseDto -> ctx.response() + .putHeader("content-type", "application/json") + .setStatusCode(200) + .end(Json.encode(responseDto)) + ); }); - router.post("/tiles/reload").blockingHandler(ctx -> { - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); + router.post("/tiles/reload").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); - if (stats == null) { - ctx.fail(new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?")); - return; - } - final var newStats = TileStatisticHolder.resetRangesWithIncrementedVersion(stats, chunkedFile.getResolutions().length); - map.put("TileStatisticHolder", newStats); - ctx.response().setStatusCode(200).end(Json.encode(Map.of("version", newStats.versionCounter().get()))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); + if (stats == null) { + throw new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?"); + } + final var newStats = TileStatisticHolder.resetRangesWithIncrementedVersion(stats, chunkedFile.getResolutions().length); + map.put("TileStatisticHolder", newStats); + scheduler.bumpGeneration(RequestTaskScheduler.CancellationDomain.TILE); + scheduler.bumpGeneration(RequestTaskScheduler.CancellationDomain.TRACK); + return Map.of("version", newStats.versionCounter().get()); + }, + result -> ctx.response().setStatusCode(200).end(Json.encode(result)) + ); }); router.get("/get_tile").handler(ctx -> { - log.debug("Entered non-blocking handler"); - ctx.next(); - }).blockingHandler(ctx -> { - log.debug("Entered blockingHandler"); - - final var row = Long.parseLong(ctx.request().getParam("row", "0")); - final var col = Long.parseLong(ctx.request().getParam("col", "0")); - final var requestedVersion = Long.parseLong(ctx.request().getParam("version", "0")); - final int tileHeight; - final int tileWidth; - final var format = TileFormat.valueOf(ctx.request().getParam("format", "JSON_PNG_WITH_RANGES")); - - log.debug("Got parameters"); - - final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); - - log.debug("Got map"); - final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); - if (chunkedFileWrapper == null) { - ctx.fail(new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?")); - return; - } - final var chunkedFile = chunkedFileWrapper.getChunkedFile(); - log.debug("Got ChunkedFile from map"); - final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); - if (visualizationOptionsWrapper == null) { - ctx.fail(new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?")); + final var scheduler = getScheduler(ctx); + if (scheduler == null) { return; } - final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); - - final var requestedBpResolutionParam = ctx.request().getParam("bpResolution"); - final int level; - if (requestedBpResolutionParam != null) { - final var requestedBpResolution = Long.parseLong(requestedBpResolutionParam); - final var resolutionOrder = chunkedFile.getResolutionToIndex().get(requestedBpResolution); - if (resolutionOrder == null) { - ctx.fail(new RuntimeException("Requested bpResolution is not present in opened file: " + requestedBpResolution)); - return; - } - level = resolutionOrder; - } else { - level = chunkedFile.getResolutions().length - Integer.parseInt(ctx.request().getParam("level", "0")); - } + final var format = TileFormat.valueOf(ctx.request().getParam("format", "JSON_PNG_WITH_RANGES")); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.TILE, + RequestTaskScheduler.CancellationDomain.TILE, + () -> computeTileResponse(ctx), + response -> { + ctx.response().putHeader("content-type", response.contentType()); + if (response.jsonBody() != null) { + ctx.response().end(response.jsonBody()); + } else { + ctx.response().end(response.binaryBody()); + } + }, + () -> respondCancelledTile(ctx, format) + ); + }); + } - final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); - if (stats == null) { - ctx.fail(new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?")); - return; - } + private TileResponsePayload computeTileResponse(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final var row = Long.parseLong(ctx.request().getParam("row", "0")); + final var col = Long.parseLong(ctx.request().getParam("col", "0")); + final var requestedVersion = Long.parseLong(ctx.request().getParam("version", "0")); + final int tileHeight; + final int tileWidth; + final var format = TileFormat.valueOf(ctx.request().getParam("format", "JSON_PNG_WITH_RANGES")); - var currentVersion = stats.versionCounter().get(); - long version = requestedVersion; - if (version < currentVersion) { - log.debug(String.format("Current version is %d and request version is %d; serving with current version", currentVersion, version)); - version = currentVersion; - } - do { - currentVersion = stats.versionCounter().get(); - } while ((currentVersion < version) && !stats.versionCounter().compareAndSet(currentVersion, version)); + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); + if (chunkedFileWrapper == null) { + throw new RuntimeException("Chunked file is not present in the local map, maybe the file is not yet opened?"); + } + final var chunkedFile = chunkedFileWrapper.getChunkedFile(); + final var visualizationOptionsWrapper = ((ShareableWrappers.SimpleVisualizationOptionsWrapper) (map.get("visualizationOptions"))); + if (visualizationOptionsWrapper == null) { + throw new RuntimeException("Visualization options are not present in the local map, maybe the file is not yet opened?"); + } + final var options = visualizationOptionsWrapper.getSimpleVisualizationOptions(); - final long startRowPx, startColPx, endRowPx, endColPx; - if (format == TileFormat.PNG_BY_PIXELS) { - startRowPx = row; - startColPx = col; - endRowPx = startRowPx + Long.parseLong(ctx.request().getParam("rows", "0")); - endColPx = startColPx + Long.parseLong(ctx.request().getParam("cols", "0")); - tileHeight = (int) (endRowPx - startRowPx); - tileWidth = (int) (endColPx - startColPx); - } else { - tileHeight = Integer.parseInt(ctx.request().getParam("tile_size", "256")); - tileWidth = Integer.parseInt(ctx.request().getParam("tile_size", "256")); - startRowPx = row * tileHeight; - endRowPx = (row + 1) * tileHeight; - startColPx = col * tileWidth; - endColPx = (col + 1) * tileWidth; + final var requestedBpResolutionParam = ctx.request().getParam("bpResolution"); + final int level; + if (requestedBpResolutionParam != null) { + final var requestedBpResolution = Long.parseLong(requestedBpResolutionParam); + final var resolutionOrder = chunkedFile.getResolutionToIndex().get(requestedBpResolution); + if (resolutionOrder == null) { + throw new RuntimeException("Requested bpResolution is not present in opened file: " + requestedBpResolution); } + level = resolutionOrder; + } else { + level = chunkedFile.getResolutions().length - Integer.parseInt(ctx.request().getParam("level", "0")); + } - final var matrixWithWeights = chunkedFile.matrixQueries().getSubmatrix(ResolutionDescriptor.fromResolutionOrder(level), startRowPx, startColPx, endRowPx, endColPx, true); - final var image = chunkedFile.tileVisualizationProcessor().visualizeTile(matrixWithWeights, options); - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); + if (stats == null) { + throw new RuntimeException("Tile statistics is not present in the local map, maybe the file is not yet opened?"); + } - log.debug("Created byte stream"); + var currentVersion = stats.versionCounter().get(); + long version = requestedVersion; + if (version < currentVersion) { + version = currentVersion; + } + do { + currentVersion = stats.versionCounter().get(); + } while ((currentVersion < version) && !stats.versionCounter().compareAndSet(currentVersion, version)); - /* - try (final var pool = Executors.newSingleThreadExecutor()) { - pool.submit(() -> { - // Update signal ranges - final var tileSummary = Arrays.stream(normalized).flatMapToLong(Arrays::stream).summaryStatistics(); - final var tileMinimum = tileSummary.getMin(); - final var tileMaximum = tileSummary.getMax(); + final long startRowPx, startColPx, endRowPx, endColPx; + if (format == TileFormat.PNG_BY_PIXELS) { + startRowPx = row; + startColPx = col; + endRowPx = startRowPx + Long.parseLong(ctx.request().getParam("rows", "0")); + endColPx = startColPx + Long.parseLong(ctx.request().getParam("cols", "0")); + tileHeight = (int) (endRowPx - startRowPx); + tileWidth = (int) (endColPx - startColPx); + } else { + tileHeight = Integer.parseInt(ctx.request().getParam("tile_size", "256")); + tileWidth = Integer.parseInt(ctx.request().getParam("tile_size", "256")); + startRowPx = row * tileHeight; + endRowPx = (row + 1) * tileHeight; + startColPx = col * tileWidth; + endColPx = (col + 1) * tileWidth; + } - long oldMinimumDoubleBits; - do { - oldMinimumDoubleBits = stats.minimumsAtResolutionDoubleBits().get(level); - } while (Double.longBitsToDouble(oldMinimumDoubleBits) > tileMinimum && !stats.minimumsAtResolutionDoubleBits().compareAndSet(level, oldMinimumDoubleBits, Double.doubleToLongBits(tileMinimum))); + final var matrixWithWeights = chunkedFile.matrixQueries().getSubmatrix(ResolutionDescriptor.fromResolutionOrder(level), startRowPx, startColPx, endRowPx, endColPx, true); + final var image = chunkedFile.tileVisualizationProcessor().visualizeTile(matrixWithWeights, options); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - long oldMaximumDoubleBits; - do { - oldMaximumDoubleBits = stats.maximumsAtResolutionDoubleBits().get(level); - } while (Double.longBitsToDouble(oldMaximumDoubleBits) < tileMaximum && !stats.maximumsAtResolutionDoubleBits().compareAndSet(level, oldMaximumDoubleBits, Double.doubleToLongBits(tileMaximum))); - }); + try { + if (format == TileFormat.JSON_PNG_WITH_RANGES) { + ImageIO.write(image, "png", baos); + final byte[] base64 = Base64.getEncoder().encode(baos.toByteArray()); + final String base64image = new String(base64); + final var result = new TileWithRanges( + String.format("data:image/png;base64,%s", base64image), + buildSignalRanges(stats, chunkedFile) + ); + return new TileResponsePayload("application/json", Json.encode(result), null); } - */ + ImageIO.write(image, "png", baos); + return new TileResponsePayload("image/png", null, Buffer.buffer(baos.toByteArray())); + } catch (final IOException e) { + throw new RuntimeException("Cannot write tile image", e); + } + } - try { - if (format == TileFormat.JSON_PNG_WITH_RANGES) { - ImageIO.write(image, "png", baos); // convert BufferedImage to byte array + private void respondCancelledTile(final @NotNull io.vertx.ext.web.RoutingContext ctx, + final @NotNull TileFormat format) { + if (format == TileFormat.JSON_PNG_WITH_RANGES) { + final var map = vertx.sharedData().getLocalMap("hict_server"); + final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) map.get("chunkedFile")); + final var stats = (TileStatisticHolder) map.get("TileStatisticHolder"); + final var ranges = (chunkedFileWrapper != null && stats != null) + ? buildSignalRanges(stats, chunkedFileWrapper.getChunkedFile()) + : new TileSignalRanges(Collections.emptyMap(), Collections.emptyMap()); + final var result = new TileWithRanges("data:image/png;base64," + TRANSPARENT_PNG_BASE64, ranges); + ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(result)); + return; + } + ctx.response() + .putHeader("content-type", "image/png") + .end(Buffer.buffer(TRANSPARENT_PNG_BYTES)); + } - final byte[] base64 = Base64.getEncoder().encode(baos.toByteArray()); - final String base64image = new String(base64); - final var result = new TileWithRanges( - String.format("data:image/png;base64,%s", base64image), - new TileSignalRanges( - IntStream.range(0, chunkedFile.getResolutions().length).boxed().map( - lvl -> Map.entry(chunkedFile.getResolutions().length - lvl, Double.longBitsToDouble(stats.minimumsAtResolutionDoubleBits().get(lvl))) - ).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)), - IntStream.range(0, chunkedFile.getResolutions().length).boxed().map( - lvl -> Map.entry(chunkedFile.getResolutions().length - lvl, Double.longBitsToDouble(stats.maximumsAtResolutionDoubleBits().get(lvl))) - ).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)) - ) - ); - log.debug("Wrote stream to buffer"); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(result)); - } else { - ImageIO.write(image, "png", baos); // convert BufferedImage to byte array - log.debug("Wrote stream to buffer"); - ctx.response() - .putHeader("content-type", "image/png") - .end(Buffer.buffer(baos.toByteArray())); - } - log.debug("Response"); - } catch (final IOException e) { - log.error("Cannot write tile image: " + e.getMessage()); - } - }); + private TileSignalRanges buildSignalRanges(final @NotNull TileStatisticHolder stats, + final @NotNull ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile chunkedFile) { + return new TileSignalRanges( + IntStream.range(0, chunkedFile.getResolutions().length).boxed().map( + lvl -> Map.entry(chunkedFile.getResolutions().length - lvl, Double.longBitsToDouble(stats.minimumsAtResolutionDoubleBits().get(lvl))) + ).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)), + IntStream.range(0, chunkedFile.getResolutions().length).boxed().map( + lvl -> Map.entry(chunkedFile.getResolutions().length - lvl, Double.longBitsToDouble(stats.maximumsAtResolutionDoubleBits().get(lvl))) + ).collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)) + ); + } + + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); } @@ -284,4 +330,9 @@ public record TileSignalRanges(@NotNull Map<@NotNull Integer, @NotNull Double> l public record TileWithRanges(@NotNull String image, @NotNull TileSignalRanges ranges) { } + + private record TileResponsePayload(@NotNull String contentType, + String jsonBody, + Buffer binaryBody) { + } } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java index 2e30581..e7eff0b 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/tracks/TrackHandlersHolder.java @@ -25,7 +25,6 @@ package ru.itmo.ctlab.hict.hict_server.handlers.tracks; import io.vertx.core.Vertx; -import io.vertx.core.WorkerExecutor; import io.vertx.core.json.Json; import io.vertx.core.shareddata.LocalMap; import io.vertx.ext.web.Router; @@ -34,40 +33,45 @@ import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_library.domain.QueryLengthUnit; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; +import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; public class TrackHandlersHolder extends HandlersHolder { private final Vertx vertx; - private final WorkerExecutor trackQueryWorkerExecutor; public TrackHandlersHolder(final Vertx vertx) { this.vertx = vertx; - final var workerPoolSize = Math.max(2, Math.min(8, Runtime.getRuntime().availableProcessors() / 2)); - this.trackQueryWorkerExecutor = vertx.createSharedWorkerExecutor( - "hict-tracks-query-worker", - workerPoolSize, - 10, - TimeUnit.MINUTES - ); } @Override public void addHandlersToRouter(final @NotNull Router router) { - router.post("/tracks/list_files").blockingHandler(ctx -> { + router.post("/tracks/list_files").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var manager = getTrackManager(ctx); if (manager == null) { return; } - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(manager.listTrackFiles())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + manager::listTrackFiles, + files -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(files)) + ); }); - router.post("/tracks/open").blockingHandler(ctx -> { + router.post("/tracks/open").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var request = ctx.body().asJsonObject(); final var filename = request.getString("filename"); if (filename == null || filename.isBlank()) { @@ -78,33 +82,53 @@ public void addHandlersToRouter(final @NotNull Router router) { if (manager == null) { return; } - final var summary = manager.openTrack( - filename, - request.getString("name"), - request.getString("color") - ); final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); final var chunkedFile = extractChunkedFile(map, ctx); if (chunkedFile == null) { return; } - manager.startPrecompute(chunkedFile, summary.getTrackId(), false); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(summary)); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + final var summary = manager.openTrack( + filename, + request.getString("name"), + request.getString("color") + ); + manager.startPrecompute(chunkedFile, summary.getTrackId(), false); + return summary; + }, + summary -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(summary)) + ); }); - router.post("/tracks/list").blockingHandler(ctx -> { + router.post("/tracks/list").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var manager = getTrackManager(ctx); if (manager == null) { return; } - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(manager.listTracks())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + manager::listTracks, + tracks -> ctx.response().putHeader("content-type", "application/json").end(Json.encode(tracks)) + ); }); - router.post("/tracks/update").blockingHandler(ctx -> { + router.post("/tracks/update").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var request = ctx.body().asJsonObject(); final var trackId = request.getString("trackId"); if (trackId == null || trackId.isBlank()) { @@ -115,20 +139,29 @@ public void addHandlersToRouter(final @NotNull Router router) { if (manager == null) { return; } - final var updated = manager.updateTrack( - trackId, - request.containsKey("visible") ? request.getBoolean("visible") : null, - request.getString("color"), - request.getString("name"), - request.getString("renderMode"), - request.getString("aggregationMode") + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> manager.updateTrack( + trackId, + request.containsKey("visible") ? request.getBoolean("visible") : null, + request.getString("color"), + request.getString("name"), + request.getString("renderMode"), + request.getString("aggregationMode") + ), + updated -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(updated)) ); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(updated)); }); - router.post("/tracks/remove").blockingHandler(ctx -> { + router.post("/tracks/remove").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var request = ctx.body().asJsonObject(); final var trackId = request.getString("trackId"); if (trackId == null || trackId.isBlank()) { @@ -139,23 +172,45 @@ public void addHandlersToRouter(final @NotNull Router router) { if (manager == null) { return; } - manager.removeTrack(trackId); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(Map.of("status", "removed", "trackId", trackId))); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.ASSEMBLY, + null, + () -> { + manager.removeTrack(trackId); + return Map.of("status", "removed", "trackId", trackId); + }, + response -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(response)) + ); }); - router.post("/tracks/precompute/status").blockingHandler(ctx -> { + router.post("/tracks/precompute/status").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var manager = getTrackManager(ctx); if (manager == null) { return; } - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(manager.getPrecomputeStatus())); + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.UI_UX, + null, + manager::getPrecomputeStatus, + status -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(status)) + ); }); - router.post("/tracks/precompute/start").blockingHandler(ctx -> { + router.post("/tracks/precompute/start").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var request = ctx.body().asJsonObject(); final var manager = getTrackManager(ctx); if (manager == null) { @@ -166,17 +221,29 @@ public void addHandlersToRouter(final @NotNull Router router) { if (chunkedFile == null) { return; } - final var status = manager.startPrecompute( - chunkedFile, - request.getString("trackId"), - request.getBoolean("force", false) + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.TRACK, + RequestTaskScheduler.CancellationDomain.TRACK, + () -> manager.startPrecompute( + chunkedFile, + request.getString("trackId"), + request.getBoolean("force", false) + ), + status -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(status)), + () -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(Map.of("status", "cancelled"))) ); - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(status)); }); router.post("/tracks/query_1d").handler(ctx -> { + final var scheduler = getScheduler(ctx); + if (scheduler == null) { + return; + } final var request = ctx.body().asJsonObject(); final var widthPx = request.getInteger("widthPx", 512); final var bpResolution = request.getLong("bpResolution", 1L); @@ -193,23 +260,27 @@ public void addHandlersToRouter(final @NotNull Router router) { final var resolvedUnits = resolveUnits(request); final var start = resolveStart(request, resolvedUnits); final var end = resolveEnd(request, resolvedUnits, start + 1L); - this.trackQueryWorkerExecutor.executeBlocking(promise -> { - try { - promise.complete(manager.queryVisibleTracks(chunkedFile, start, end, widthPx, bpResolution, resolvedUnits)); - } catch (final Throwable t) { - promise.fail(t); - } - }, - false, - ar -> { - if (ar.failed()) { - ctx.fail(ar.cause()); - return; - } - ctx.response() - .putHeader("content-type", "application/json") - .end(Json.encode(ar.result())); - }); + + scheduler.submit( + ctx, + RequestTaskScheduler.RequestPriority.TRACK, + RequestTaskScheduler.CancellationDomain.TRACK, + () -> manager.queryVisibleTracks(chunkedFile, start, end, widthPx, bpResolution, resolvedUnits), + result -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(result)), + () -> ctx.response() + .putHeader("content-type", "application/json") + .end(Json.encode(new Track1DManager.QueryResult( + 0L, + 0L, + start, + Math.max(start + 1L, end), + widthPx, + bpResolution, + List.of() + ))) + ); }); } @@ -271,6 +342,16 @@ private Track1DManager getTrackManager(final @NotNull io.vertx.ext.web.RoutingCo return managerWrapper.getTrack1DManager(); } + private RequestTaskScheduler getScheduler(final @NotNull io.vertx.ext.web.RoutingContext ctx) { + final @NotNull @NonNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var wrapper = (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (wrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return null; + } + return wrapper.getRequestTaskScheduler(); + } + private ChunkedFile extractChunkedFile(final @NotNull LocalMap map, final @NotNull io.vertx.ext.web.RoutingContext ctx) { final var chunkedFileWrapper = ((ShareableWrappers.ChunkedFileWrapper) (map.get("chunkedFile"))); if (chunkedFileWrapper == null) { diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java index e4a9b79..a6f6d9a 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/util/shareable/ShareableWrappers.java @@ -30,6 +30,7 @@ import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_library.chunkedfile.ChunkedFile; import ru.itmo.ctlab.hict.hict_library.visualization.SimpleVisualizationOptions; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; import ru.itmo.ctlab.hict.hict_server.tracks.Track1DManager; import java.nio.file.Path; @@ -58,4 +59,10 @@ public static class SimpleVisualizationOptionsWrapper implements Shareable { public static class Track1DManagerWrapper implements Shareable { private final @NotNull Track1DManager track1DManager; } + + @Getter + @RequiredArgsConstructor + public static class RequestTaskSchedulerWrapper implements Shareable { + private final @NotNull RequestTaskScheduler requestTaskScheduler; + } } diff --git a/version.txt b/version.txt index f17dda7..47207a6 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.84-aa7deb2-webui_f21ba84 \ No newline at end of file +1.0.86-ad8acb7-webui_543dcbb \ No newline at end of file From 87e372b1fbf75474ef918766ea33d8c3e41c78a3 Mon Sep 17 00:00:00 2001 From: Alexander Serdyukov Date: Sat, 21 Mar 2026 06:09:00 +0400 Subject: [PATCH 5/5] Tracks stability improved, added load-status panel --- .../concurrent/RequestTaskScheduler.java | 87 ++++ .../handlers/fileop/FileOpHandlersHolder.java | 4 + .../handlers/info/InfoHandlersHolder.java | 18 + .../names/NameMappingHandlersHolder.java | 4 + .../ScaffoldingOpHandlersHolder.java | 4 + .../handlers/tiles/TileHandlersHolder.java | 4 + .../tools/ConversionCliLauncher.java | 27 +- .../ctlab/hict/hict_server/tools/HictCli.java | 18 +- .../hict_server/tracks/Track1DManager.java | 423 +++++++++++++++--- version.txt | 2 +- 10 files changed, 523 insertions(+), 68 deletions(-) diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java index 9abd9be..f7c2173 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/concurrent/RequestTaskScheduler.java @@ -32,6 +32,7 @@ import org.jetbrains.annotations.Nullable; import java.util.EnumMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.concurrent.CancellationException; @@ -56,6 +57,8 @@ public final class RequestTaskScheduler implements AutoCloseable { private final @NotNull EnumMap generations; private final @NotNull EnumMap>>> trackedTasks; private final @NotNull Semaphore globalElasticPermits; + private final int totalMaxWorkers; + private final int reservedMinWorkers; @Getter public static final class PoolSizing { @@ -98,6 +101,37 @@ public enum CancellationDomain { EXPORT } + public record PoolDiagnostics( + int corePoolSize, + int maxPoolSize, + int currentPoolSize, + int largestPoolSize, + int activeCount, + int queueSize, + int queueCapacity, + long completedTaskCount, + long taskCount + ) { + } + + public record CancellationDomainDiagnostics( + long currentGeneration, + int trackedTaskCount, + @NotNull Map trackedTasksByGeneration + ) { + } + + public record SchedulerDiagnostics( + long timestampMs, + int totalMaxWorkers, + int reservedMinWorkers, + int elasticWorkersInUse, + int elasticWorkersAvailable, + @NotNull Map pools, + @NotNull Map cancellationDomains + ) { + } + @FunctionalInterface public interface ThrowingSupplier { T get() throws Exception; @@ -132,6 +166,8 @@ public RequestTaskScheduler(final @NotNull Vertx vertx, final int totalMaxWorkers = Math.max(minTotal, config.totalMaxWorkers()); final int elasticBudget = Math.max(0, totalMaxWorkers - minTotal); this.globalElasticPermits = new Semaphore(elasticBudget, true); + this.totalMaxWorkers = totalMaxWorkers; + this.reservedMinWorkers = minTotal; for (final var domain : CancellationDomain.values()) { this.generations.put(domain, new AtomicLong(0L)); @@ -259,6 +295,57 @@ public void bumpAssemblyGeneration() { bumpGeneration(CancellationDomain.EXPORT); } + public @NotNull SchedulerDiagnostics diagnosticsSnapshot() { + final var poolSnapshots = new EnumMap(RequestPriority.class); + for (final var entry : this.pools.entrySet()) { + final var pool = entry.getValue(); + final int queueSize = pool.getQueue().size(); + final int queueCapacity = queueSize + pool.getQueue().remainingCapacity(); + poolSnapshots.put( + entry.getKey(), + new PoolDiagnostics( + pool.getCorePoolSize(), + pool.getMaximumPoolSize(), + pool.getPoolSize(), + pool.getLargestPoolSize(), + pool.getActiveCount(), + queueSize, + queueCapacity, + pool.getCompletedTaskCount(), + pool.getTaskCount() + ) + ); + } + final var cancellationSnapshots = new EnumMap(CancellationDomain.class); + for (final var domain : CancellationDomain.values()) { + final var trackedByGeneration = new LinkedHashMap(); + int trackedCount = 0; + for (final var entry : this.trackedTasks.get(domain).entrySet()) { + final int size = entry.getValue().size(); + trackedCount += size; + trackedByGeneration.put(entry.getKey(), size); + } + cancellationSnapshots.put( + domain, + new CancellationDomainDiagnostics( + this.currentGeneration(domain), + trackedCount, + trackedByGeneration + ) + ); + } + final int elasticAvailable = this.globalElasticPermits.availablePermits(); + return new SchedulerDiagnostics( + System.currentTimeMillis(), + this.totalMaxWorkers, + this.reservedMinWorkers, + Math.max(0, this.totalMaxWorkers - this.reservedMinWorkers - elasticAvailable), + elasticAvailable, + poolSnapshots, + cancellationSnapshots + ); + } + public long currentGeneration(final @NotNull CancellationDomain domain) { return this.generations.get(domain).get(); } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java index 31170eb..909ab15 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/fileop/FileOpHandlersHolder.java @@ -439,6 +439,10 @@ public void addHandlersToRouter(final @NotNull Router router) { if (schedulerWrapper != null) { schedulerWrapper.getRequestTaskScheduler().bumpAssemblyGeneration(); } + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().invalidateInMemoryCache(); + } return AssemblyInfoDTO.generateFromChunkedFile(chunkedFile); }, response -> ctx.response().end(Json.encode(response)) diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/info/InfoHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/info/InfoHandlersHolder.java index 4e2b504..f180fc1 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/info/InfoHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/info/InfoHandlersHolder.java @@ -2,9 +2,12 @@ import io.vertx.core.Vertx; import io.vertx.core.json.Json; +import io.vertx.core.shareddata.LocalMap; import io.vertx.ext.web.Router; import org.jetbrains.annotations.NotNull; import ru.itmo.ctlab.hict.hict_server.HandlersHolder; +import ru.itmo.ctlab.hict.hict_server.concurrent.RequestTaskScheduler; +import ru.itmo.ctlab.hict.hict_server.util.shareable.ShareableWrappers; import java.io.BufferedReader; import java.io.InputStream; @@ -34,6 +37,21 @@ public void addHandlersToRouter(final @NotNull Router router) { "webuiVersion", webuiVersion ))); }); + + router.post("/diagnostics/workers").handler(ctx -> { + final @NotNull LocalMap map = this.vertx.sharedData().getLocalMap("hict_server"); + final var schedulerWrapper = + (ShareableWrappers.RequestTaskSchedulerWrapper) map.get(RequestTaskScheduler.LOCAL_MAP_KEY); + if (schedulerWrapper == null) { + ctx.fail(new IllegalStateException("Request scheduler is not initialized")); + return; + } + final var snapshot = schedulerWrapper.getRequestTaskScheduler().diagnosticsSnapshot(); + ctx.response() + .putHeader("content-type", "application/json") + .setStatusCode(200) + .end(Json.encode(snapshot)); + }); } private @NotNull String readVersion() { diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java index cd710bd..408761a 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/names/NameMappingHandlersHolder.java @@ -181,6 +181,10 @@ private long incrementVersionAndResetTileStats(final @NotNull ChunkedFile chunke final var newStats = TileStatisticHolder.resetRangesWithIncrementedVersion(stats, chunkedFile.getResolutions().length); map.put("TileStatisticHolder", newStats); scheduler.bumpAssemblyGeneration(); + final var trackManagerWrapper = (ShareableWrappers.Track1DManagerWrapper) map.get("Track1DManager"); + if (trackManagerWrapper != null) { + trackManagerWrapper.getTrack1DManager().invalidateInMemoryCache(); + } return newStats.versionCounter().get(); } diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java index 2875f05..2c8db0c 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/handlers/operations/ScaffoldingOpHandlersHolder.java @@ -232,6 +232,10 @@ private long incrementVersionAndResetTileStats(final @NotNull LocalMap ctx.response().setStatusCode(200).end(Json.encode(result)) diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/ConversionCliLauncher.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/ConversionCliLauncher.java index 6747abb..d161187 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/ConversionCliLauncher.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/ConversionCliLauncher.java @@ -9,8 +9,11 @@ import java.util.Arrays; import java.util.List; import java.util.function.Consumer; +import java.util.regex.Pattern; public class ConversionCliLauncher { + private static final Pattern OVERALL_PROGRESS_PATTERN = Pattern.compile("Overall progress:\\s*(\\d+)%"); + private static final Pattern LOCAL_PROGRESS_PATTERN = Pattern.compile("^[^\\n]*:\\s*(\\d+)%\\s*\\("); static { HDF5LibraryInitializer.initializeHDF5Library(); @@ -24,6 +27,7 @@ public static void main(String[] args) throws Exception { final var command = args[0]; final var parser = new ArgParser(Arrays.copyOfRange(args, 1, args.length)); + final var verbose = parser.flag("verbose"); final var options = new ConversionOptions( Path.of(parser.require("input")), Path.of(parser.require("output")), @@ -37,16 +41,27 @@ public static void main(String[] args) throws Exception { ); switch (command) { - case "hict-to-mcool" -> new HictToMcoolConverter().convert(options, stdoutLogger()); - case "mcool-to-hict" -> new McoolToHictConverter().convert(options, stdoutLogger()); + case "hict-to-mcool" -> new HictToMcoolConverter().convert(options, stdoutLogger(verbose)); + case "mcool-to-hict" -> new McoolToHictConverter().convert(options, stdoutLogger(verbose)); default -> throw new IllegalArgumentException("Unknown command: " + command); } } - private static Consumer stdoutLogger() { + private static Consumer stdoutLogger(final boolean verbose) { return message -> { + if (!verbose) { + return; + } synchronized (System.out) { - System.out.println(message); + final var overall = OVERALL_PROGRESS_PATTERN.matcher(message); + final var local = LOCAL_PROGRESS_PATTERN.matcher(message); + if (overall.find()) { + System.out.println("[TOTAL " + overall.group(1) + "%] " + message); + } else if (local.find()) { + System.out.println("[LOCAL " + local.group(1) + "%] " + message); + } else { + System.out.println(message); + } System.out.flush(); } }; @@ -54,8 +69,8 @@ private static Consumer stdoutLogger() { private static void printHelp() { System.out.println("Usage:"); - System.out.println(" hict-to-mcool --input= --output= [--resolutions=10000,50000] [--compression=0..9 (default: 6)] [--compression-algorithm=deflate|zstd|lzf] [--chunk-size=8192] [--agp=foo.agp --apply-agp] [--parallelism=N]"); - System.out.println(" mcool-to-hict --input= --output= [--resolutions=10000,50000] [--compression=0..9 (default: 6)] [--compression-algorithm=deflate|zstd|lzf] [--chunk-size=8192] [--parallelism=N]"); + System.out.println(" hict-to-mcool --input= --output= [--resolutions=10000,50000] [--compression=0..9 (default: 6)] [--compression-algorithm=deflate|zstd|lzf] [--chunk-size=8192] [--agp=foo.agp --apply-agp] [--parallelism=N] [--verbose]"); + System.out.println(" mcool-to-hict --input= --output= [--resolutions=10000,50000] [--compression=0..9 (default: 6)] [--compression-algorithm=deflate|zstd|lzf] [--chunk-size=8192] [--parallelism=N] [--verbose]"); } private record ArgParser(String[] args) { diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/HictCli.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/HictCli.java index f8f9d78..9e3ac74 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/HictCli.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tools/HictCli.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.function.Consumer; +import java.util.regex.Pattern; @Command( name = "hict", @@ -109,6 +110,9 @@ public void run() { } abstract static class BaseConvert implements Callable { + private static final Pattern OVERALL_PROGRESS_PATTERN = Pattern.compile("Overall progress:\\s*(\\d+)%"); + private static final Pattern LOCAL_PROGRESS_PATTERN = Pattern.compile("^[^\\n]*:\\s*(\\d+)%\\s*\\("); + @Option(names = {"-i", "--input"}, required = true, description = "Input file path.") Path input; @@ -158,9 +162,21 @@ ConversionOptions toOptions(String agpPath, boolean applyAgp) { } Consumer stdoutLogger() { + final boolean verboseEnabled = Boolean.parseBoolean(System.getProperty("HICT_VERBOSE", "false")); return message -> { + if (!verboseEnabled) { + return; + } synchronized (System.out) { - System.out.println(message); + final var overall = OVERALL_PROGRESS_PATTERN.matcher(message); + final var local = LOCAL_PROGRESS_PATTERN.matcher(message); + if (overall.find()) { + System.out.println("[TOTAL " + overall.group(1) + "%] " + message); + } else if (local.find()) { + System.out.println("[LOCAL " + local.group(1) + "%] " + message); + } else { + System.out.println(message); + } System.out.flush(); } }; diff --git a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java index dc394ee..41e2e62 100644 --- a/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java +++ b/src/main/java/ru/itmo/ctlab/hict/hict_server/tracks/Track1DManager.java @@ -24,7 +24,9 @@ package ru.itmo.ctlab.hict.hict_server.tracks; +import ch.systemsx.cisd.hdf5.HDF5FloatStorageFeatures; import ch.systemsx.cisd.hdf5.HDF5Factory; +import ch.systemsx.cisd.hdf5.HDF5IntStorageFeatures; import htsjdk.samtools.SAMRecord; import htsjdk.samtools.SAMSequenceDictionary; import htsjdk.samtools.SamReader; @@ -55,7 +57,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; @@ -77,7 +78,13 @@ public class Track1DManager { private static final int MAX_FEATURES_PER_QUERY = 250_000; private static final String PRECOMPUTE_CACHE_VERSION = "1"; private static final long MAX_PRECOMPUTE_VISIBLE_PIXELS = 2_000_000L; - private static final int PRECOMPUTE_EXECUTOR_THREADS = 1; + private static final int PRECOMPUTE_JOB_THREADS = resolveThreadCount("HICT_TRACK_PRECOMPUTE_JOB_THREADS", 2); + private static final int PRECOMPUTE_WORKER_THREADS = resolveThreadCount( + "HICT_TRACK_PRECOMPUTE_WORKER_THREADS", + Math.max(2, Runtime.getRuntime().availableProcessors()) + ); + private static final int PRECOMPUTE_DATASET_CHUNK_SIZE = 8192; + private static final int PRECOMPUTE_COMPRESSION_LEVEL = 4; private static final long PRECOMPUTE_STATUS_TTL_MS = 15L * 60_000L; private final @NotNull Path dataDirectory; @@ -85,7 +92,9 @@ public class Track1DManager { private final @NotNull ReadWriteLock lock = new ReentrantReadWriteLock(); private final @NotNull LinkedHashMap tracks = new LinkedHashMap<>(); private final @NotNull AtomicLong trackCounter = new AtomicLong(0L); - private final @NotNull ExecutorService precomputeExecutor; + private final @NotNull ExecutorService precomputeJobExecutor; + private final @NotNull ExecutorService precomputeWorkerExecutor; + private final @NotNull ExecutorService precomputeWriterExecutor; private final @NotNull ConcurrentHashMap precomputedSeriesCache = new ConcurrentHashMap<>(); private final @NotNull ConcurrentHashMap precomputeRuntimeByTrackId = new ConcurrentHashMap<>(); private volatile @NotNull Map linkedFastaAliasesBySource = Map.of(); @@ -104,17 +113,49 @@ public Track1DManager(final @NotNull Path dataDirectory, final @Nullable Path pr } catch (final IOException e) { throw new RuntimeException("Failed to create processed directory " + this.processedDirectory, e); } - this.precomputeExecutor = Executors.newFixedThreadPool( - PRECOMPUTE_EXECUTOR_THREADS, - r -> { - final var t = new Thread(r); - t.setDaemon(true); - t.setName("hict-track-precompute"); - return t; - } + this.precomputeJobExecutor = Executors.newFixedThreadPool( + PRECOMPUTE_JOB_THREADS, + namedDaemonThreadFactory("hict-track-precompute-job") + ); + this.precomputeWorkerExecutor = Executors.newFixedThreadPool( + PRECOMPUTE_WORKER_THREADS, + namedDaemonThreadFactory("hict-track-precompute-worker") + ); + this.precomputeWriterExecutor = Executors.newSingleThreadExecutor( + namedDaemonThreadFactory("hict-track-precompute-writer") ); } + private static @NotNull ThreadFactory namedDaemonThreadFactory(final @NotNull String prefix) { + final var counter = new AtomicLong(0L); + return runnable -> { + final var thread = new Thread(runnable); + thread.setDaemon(true); + thread.setName(prefix + "-" + counter.incrementAndGet()); + return thread; + }; + } + + private static int resolveThreadCount(final @NotNull String envKey, final int defaultValue) { + final String env = System.getenv(envKey); + if (env != null && !env.isBlank()) { + try { + return Math.max(1, Integer.parseInt(env.trim())); + } catch (final NumberFormatException ignored) { + // Fallback to default. + } + } + final String property = System.getProperty(envKey); + if (property != null && !property.isBlank()) { + try { + return Math.max(1, Integer.parseInt(property.trim())); + } catch (final NumberFormatException ignored) { + // Fallback to default. + } + } + return Math.max(1, defaultValue); + } + public @NotNull List listTrackFiles() { try (final var stream = Files.walk(this.dataDirectory)) { return stream @@ -233,13 +274,19 @@ public void close() { } finally { this.lock.writeLock().unlock(); } - this.precomputeExecutor.shutdownNow(); + this.precomputeJobExecutor.shutdownNow(); + this.precomputeWorkerExecutor.shutdownNow(); + this.precomputeWriterExecutor.shutdownNow(); } public void setLinkedFastaAliasesBySource(final @Nullable Map aliases) { this.linkedFastaAliasesBySource = aliases == null ? Map.of() : Map.copyOf(aliases); } + public void invalidateInMemoryCache() { + this.precomputedSeriesCache.clear(); + } + public @NotNull TracksPrecomputeStatus getPrecomputeStatus() { final var now = System.currentTimeMillis(); this.precomputeRuntimeByTrackId.entrySet().removeIf(entry -> @@ -473,7 +520,7 @@ private void scheduleTrackPrecompute(final @NotNull ChunkedFile chunkedFile, } runtime.markQueued(); } - this.precomputeExecutor.submit(() -> runTrackPrecompute(chunkedFile, track, force, runtime)); + this.precomputeJobExecutor.submit(() -> runTrackPrecompute(chunkedFile, track, force, runtime)); } private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, @@ -481,29 +528,39 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, final boolean force, final @NotNull TrackPrecomputeRuntime runtime) { try { - final var tasks = buildPrecomputeTasks(chunkedFile, track, force); + final var sidecarPath = sidecarPathForTrackCache(chunkedFile, track); + final var tasks = buildPrecomputeTasks(chunkedFile, track, sidecarPath, force); runtime.setTotalTasks(tasks.size()); if (tasks.isEmpty()) { runtime.markFinished(); return; } - int completed = 0; + final CompletionService completionService = + new ExecutorCompletionService<>(this.precomputeWorkerExecutor); for (final var task : tasks) { + completionService.submit(() -> new ComputedPrecomputeTask(task, computePrecomputedSeries(chunkedFile, track, task))); + } + + final var writeFutures = new ArrayList>(tasks.size()); + int completed = 0; + for (int i = 0; i < tasks.size(); i++) { + final var computed = completionService.take().get(); + final var task = computed.task(); runtime.markRunning(task.bpResolution() + "bp/" + task.modeKey(), completed); final var key = new PrecomputedSeriesKey(track.trackId(), task.bpResolution(), task.assemblySignature(), task.modeKey()); - PrecomputedSeries series = this.precomputedSeriesCache.get(key); - if (series == null || force) { - series = loadPrecomputedSeriesFromSidecar(task.sidecarFile(), task.totalVisiblePixels()).orElse(null); - } - if (series == null || force) { - series = computePrecomputedSeries(chunkedFile, track, task); - persistPrecomputedSeries(task.sidecarFile(), series); - } - this.precomputedSeriesCache.put(key, series); + this.precomputedSeriesCache.put(key, computed.series()); + writeFutures.add(this.precomputeWriterExecutor.submit(() -> persistPrecomputedSeries(sidecarPath, task, computed.series()))); completed++; runtime.markTaskDone(completed); } + for (final var writeFuture : writeFutures) { + writeFuture.get(); + } runtime.markFinished(); + } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); + runtime.markFailed("Precompute interrupted"); + log.warn("Track precompute interrupted for track {}", track.trackId()); } catch (final Exception ex) { runtime.markFailed(ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage()); log.error("Failed to precompute track {}", track.trackId(), ex); @@ -512,6 +569,7 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, private @NotNull List buildPrecomputeTasks(final @NotNull ChunkedFile chunkedFile, final @NotNull TrackState track, + final @NotNull Path sidecarPath, final boolean force) { final var tasks = new ArrayList(); final var resolutions = Arrays.stream(chunkedFile.getResolutions()).boxed().sorted(Comparator.reverseOrder()).toList(); @@ -528,11 +586,20 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, if (!force && this.precomputedSeriesCache.containsKey(key)) { continue; } - final var sidecarPath = sidecarPathForVector(chunkedFile, track, bpResolution, assemblySignature, modeKey); - if (!force && Files.exists(sidecarPath)) { - continue; + if (!force) { + final var cached = loadPrecomputedSeriesFromSidecar( + sidecarPath, + bpResolution, + modeKey, + assemblySignature, + totalVisiblePixels + ); + if (cached.isPresent()) { + this.precomputedSeriesCache.put(key, cached.get()); + continue; + } } - tasks.add(new PrecomputeTask(bpResolution, modeKey, totalVisiblePixels, assemblySignature, sidecarPath)); + tasks.add(new PrecomputeTask(bpResolution, modeKey, totalVisiblePixels, assemblySignature)); } } return tasks; @@ -555,8 +622,14 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, final var key = new PrecomputedSeriesKey(track.trackId(), bpResolution, assemblySignature, modeKey); var series = this.precomputedSeriesCache.get(key); if (series == null) { - final var sidecar = sidecarPathForVector(chunkedFile, track, bpResolution, assemblySignature, modeKey); - series = loadPrecomputedSeriesFromSidecar(sidecar, totalVisiblePixels).orElse(null); + final var sidecar = sidecarPathForTrackCache(chunkedFile, track); + series = loadPrecomputedSeriesFromSidecar( + sidecar, + bpResolution, + modeKey, + assemblySignature, + totalVisiblePixels + ).orElse(null); if (series != null) { this.precomputedSeriesCache.put(key, series); } @@ -647,20 +720,26 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, } private @NotNull Optional loadPrecomputedSeriesFromSidecar(final @NotNull Path sidecarPath, + final long bpResolution, + final @NotNull String modeKey, + final @NotNull String assemblySignature, final long expectedLength) { if (!Files.exists(sidecarPath) || !Files.isRegularFile(sidecarPath)) { return Optional.empty(); } + final var groupPath = precomputeGroupPath(bpResolution, modeKey, assemblySignature); + final var valuesPath = groupPath + "/values"; + final var supportPath = groupPath + "/support"; try (final var reader = HDF5Factory.openForReading(sidecarPath.toFile())) { - if (!reader.object().isDataSet("/cache/values") || !reader.object().isDataSet("/cache/support")) { + if (!reader.object().isDataSet(valuesPath) || !reader.object().isDataSet(supportPath)) { return Optional.empty(); } - final var valuesDims = reader.object().getDataSetInformation("/cache/values").getDimensions(); + final var valuesDims = reader.object().getDataSetInformation(valuesPath).getDimensions(); if (valuesDims.length != 1 || valuesDims[0] != expectedLength) { return Optional.empty(); } - final var values = reader.float64().readArray("/cache/values"); - final var support = reader.int64().readArray("/cache/support"); + final var values = reader.float64().readArray(valuesPath); + final var support = reader.int64().readArray(supportPath); if (values.length != support.length) { return Optional.empty(); } @@ -672,30 +751,60 @@ private void runTrackPrecompute(final @NotNull ChunkedFile chunkedFile, } private void persistPrecomputedSeries(final @NotNull Path sidecarPath, + final @NotNull PrecomputeTask task, final @NotNull PrecomputedSeries series) { try { Files.createDirectories(sidecarPath.getParent()); - final var tmpPath = sidecarPath.resolveSibling(sidecarPath.getFileName() + ".tmp"); - try (final var writer = HDF5Factory.open(tmpPath.toFile())) { - if (!writer.object().isGroup("/cache")) { - writer.object().createGroup("/cache"); + final var groupPath = precomputeGroupPath(task.bpResolution(), task.modeKey(), task.assemblySignature()); + final var valuesPath = groupPath + "/values"; + final var supportPath = groupPath + "/support"; + final var chunkLen = Math.max( + 1, + Math.min(PRECOMPUTE_DATASET_CHUNK_SIZE, Math.max(series.values().length, series.support().length)) + ); + try (final var writer = HDF5Factory.open(sidecarPath.toFile())) { + ensureGroupPath(writer, groupPath); + writer.string().setAttr(groupPath, "version", PRECOMPUTE_CACHE_VERSION); + writer.int64().setAttr(groupPath, "bpResolution", task.bpResolution()); + writer.string().setAttr(groupPath, "mode", task.modeKey()); + writer.string().setAttr(groupPath, "assemblySignature", task.assemblySignature()); + writer.int64().setAttr(groupPath, "length", series.values().length); + if (!writer.object().isDataSet(valuesPath)) { + writer.float64().createArray( + valuesPath, + series.values().length, + chunkLen, + HDF5FloatStorageFeatures.createDeflation(PRECOMPUTE_COMPRESSION_LEVEL) + ); + } + if (!writer.object().isDataSet(supportPath)) { + writer.int64().createArray( + supportPath, + series.support().length, + chunkLen, + HDF5IntStorageFeatures.createDeflation(PRECOMPUTE_COMPRESSION_LEVEL) + ); } - writer.string().setAttr("/cache", "version", PRECOMPUTE_CACHE_VERSION); - writer.int64().setAttr("/cache", "length", series.values().length); - writer.float64().writeArray("/cache/values", series.values()); - writer.int64().writeArray("/cache/support", series.support()); + writer.float64().writeArrayBlockWithOffset( + valuesPath, + series.values(), + series.values().length, + 0L + ); + writer.int64().writeArrayBlockWithOffset( + supportPath, + series.support(), + series.support().length, + 0L + ); } - Files.move(tmpPath, sidecarPath, StandardCopyOption.REPLACE_EXISTING); } catch (final Exception e) { log.warn("Failed to write precomputed sidecar {}", sidecarPath, e); } } - private @NotNull Path sidecarPathForVector(final @NotNull ChunkedFile chunkedFile, - final @NotNull TrackState track, - final long bpResolution, - final @NotNull String assemblySignature, - final @NotNull String modeKey) { + private @NotNull Path sidecarPathForTrackCache(final @NotNull ChunkedFile chunkedFile, + final @NotNull TrackState track) { final var trackSource = resolveDataPath(track.sourceFile()); final var hictPath = chunkedFile.getHdfFilePath(); final String fingerprint; @@ -707,10 +816,7 @@ private void persistPrecomputedSeries(final @NotNull Path sidecarPath, String.valueOf(Files.getLastModifiedTime(trackSource).toMillis()), hictPath.toString(), String.valueOf(Files.size(hictPath)), - String.valueOf(Files.getLastModifiedTime(hictPath).toMillis()), - String.valueOf(bpResolution), - assemblySignature, - modeKey + String.valueOf(Files.getLastModifiedTime(hictPath).toMillis()) ); } catch (final IOException e) { throw new RuntimeException("Cannot build precompute fingerprint for " + track.sourceFile(), e); @@ -719,6 +825,28 @@ private void persistPrecomputedSeries(final @NotNull Path sidecarPath, return this.processedDirectory.resolve("track_precompute").resolve(fileName); } + private static @NotNull String precomputeGroupPath(final long bpResolution, + final @NotNull String modeKey, + final @NotNull String assemblySignature) { + return "/cache/resolutions/" + bpResolution + "/modes/" + modeKey + "/assemblies/" + assemblySignature; + } + + private static void ensureGroupPath(final @NotNull ch.systemsx.cisd.hdf5.IHDF5Writer writer, + final @NotNull String groupPath) { + final var parts = groupPath.split("/"); + final var current = new StringBuilder(); + for (final var part : parts) { + if (part == null || part.isBlank()) { + continue; + } + current.append('/').append(part); + final var path = current.toString(); + if (!writer.object().isGroup(path)) { + writer.object().createGroup(path); + } + } + } + private static @NotNull String computeAssemblySignature(final @NotNull List orderedSegments, final long bpResolution) { long hash = 1469598103934665603L; @@ -936,6 +1064,25 @@ private static double parseOptionalDouble(final String value, final double defau } } + private static @Nullable Double parseNullableDouble(final String value) { + if (value == null || value.isBlank()) { + return null; + } + try { + return Double.parseDouble(value); + } catch (final NumberFormatException ignored) { + return null; + } + } + + private static boolean isBedStrandToken(final String token) { + if (token == null) { + return false; + } + final var trimmed = token.trim(); + return "+".equals(trimmed) || "-".equals(trimmed) || ".".equals(trimmed); + } + private static @NotNull BufferedReader openMaybeGzipReader(final @NotNull Path filePath) throws IOException { final InputStream baseStream = Files.newInputStream(filePath); final InputStream dataStream; @@ -1445,6 +1592,24 @@ private static void accumulateReadDensityValue(final long featureStart, bigWigAggregationMode ); } + if (type == TrackType.BED) { + if (dataSource instanceof InMemoryTrackDataSource inMemoryTrackDataSource) { + return inMemoryTrackDataSource.queryBins( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + widthPx, + bpResolution + ); + } + final var projectedFeatures = dataSource.projectFeatures( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + bpResolution + ); + return aggregateCoverageFeatures(projectedFeatures, queryStartPx, queryEndPx, widthPx); + } final var projectedFeatures = dataSource.projectFeatures( sourceToAssemblySegments, queryStartPx, @@ -1510,10 +1675,12 @@ default void close() throws Exception { } private record InMemoryTrackDataSource(@NotNull Map> featuresBySource, - long featureCount) implements TrackDataSource { + long featureCount, + boolean hasSignalValues) implements TrackDataSource { static @NotNull InMemoryTrackDataSource fromBed(final @NotNull Path filePath) { final var features = new HashMap>(); long total = 0L; + boolean hasSignalValues = false; try (final BufferedReader reader = openMaybeGzipReader(filePath)) { String line; long lineNo = 0L; @@ -1532,17 +1699,35 @@ private record InMemoryTrackDataSource(@NotNull Map> if (end <= start) { continue; } - final var label = (fields.length >= 4 && !fields[3].isBlank()) ? fields[3] : null; - final var value = parseOptionalDouble(fields.length >= 5 ? fields[4] : null, 1.0d); + final var col4Numeric = fields.length >= 4 ? parseNullableDouble(fields[3]) : null; + final var col5Numeric = fields.length >= 5 ? parseNullableDouble(fields[4]) : null; + final var hasStrand = fields.length >= 6 && isBedStrandToken(fields[5]); + final String label; + final double value; + if (fields.length == 4 && col4Numeric != null) { + // BEDGraph-style row: chrom start end value + label = null; + value = Math.max(0.0d, col4Numeric); + } else if (hasStrand) { + // BED6 alignments: score is typically MAPQ-like, not quantitative signal; use unit coverage. + label = (fields.length >= 4 && !fields[3].isBlank()) ? fields[3] : null; + value = 1.0d; + } else { + label = (fields.length >= 4 && !fields[3].isBlank()) ? fields[3] : null; + value = Math.max(0.0d, col5Numeric != null ? col5Numeric : 1.0d); + } features.computeIfAbsent(sourceName, ignored -> new ArrayList<>()) - .add(new FeatureRange(start, end, Math.max(0.0d, value), label)); + .add(new FeatureRange(start, end, value, label)); + if (Math.abs(value - 1.0d) > 1e-9) { + hasSignalValues = true; + } total++; } } catch (final IOException e) { throw new RuntimeException("Failed to parse BED track " + filePath, e); } features.values().forEach(list -> list.sort(Comparator.comparingLong(FeatureRange::start))); - return new InMemoryTrackDataSource(features, total); + return new InMemoryTrackDataSource(features, total, hasSignalValues); } static @NotNull InMemoryTrackDataSource fromVcf(final @NotNull Path filePath) { @@ -1576,7 +1761,7 @@ private record InMemoryTrackDataSource(@NotNull Map> throw new RuntimeException("Failed to parse VCF track " + filePath, e); } features.values().forEach(list -> list.sort(Comparator.comparingLong(FeatureRange::start))); - return new InMemoryTrackDataSource(features, total); + return new InMemoryTrackDataSource(features, total, false); } @Override @@ -1644,6 +1829,121 @@ public long featureCountHint() { return projected; } + public @NotNull List queryBins(final @NotNull Map> sourceToAssemblySegments, + final long queryStartPx, + final long queryEndPx, + final int widthPx, + final long bpResolution) { + final var bucketCount = Math.max(1, widthPx); + final var span = Math.max(1L, queryEndPx - queryStartPx); + final var bucketSpan = Math.max(1.0d, span / (double) bucketCount); + if (this.hasSignalValues()) { + final double[] maxValues = new double[bucketCount]; + final double[] weightedSums = new double[bucketCount]; + final double[] overlapSums = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + forEachProjectedFeature( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + bpResolution, + feature -> accumulateBigWigValue( + feature.startPx(), + feature.endPx(), + feature.value(), + queryStartPx, + queryEndPx, + bucketSpan, + maxValues, + weightedSums, + overlapSums, + counts + ) + ); + return finalizeBigWigBins( + queryStartPx, + queryEndPx, + bucketSpan, + maxValues, + weightedSums, + overlapSums, + counts, + BigWigAggregationMode.MAX + ); + } + final double[] values = new double[bucketCount]; + final long[] counts = new long[bucketCount]; + forEachProjectedFeature( + sourceToAssemblySegments, + queryStartPx, + queryEndPx, + bpResolution, + feature -> accumulateCoverageValue( + feature.startPx(), + feature.endPx(), + queryStartPx, + queryEndPx, + bucketSpan, + values, + counts + ) + ); + return finalizeBins(queryStartPx, queryEndPx, bucketSpan, values, counts); + } + + private void forEachProjectedFeature(final @NotNull Map> sourceToAssemblySegments, + final long queryStartPx, + final long queryEndPx, + final long bpResolution, + final @NotNull java.util.function.Consumer consumer) { + for (final var entry : this.featuresBySource.entrySet()) { + final var sourceName = entry.getKey(); + final var sourceFeatures = entry.getValue(); + if (sourceFeatures.isEmpty()) { + continue; + } + final var assemblySegments = sourceToAssemblySegments.get(sourceName); + if (assemblySegments == null || assemblySegments.isEmpty()) { + continue; + } + for (final var segment : assemblySegments) { + final var sourceIntervalOptional = mapVisiblePxIntervalToSegmentSource( + segment, + queryStartPx, + queryEndPx, + bpResolution + ); + if (sourceIntervalOptional.isEmpty()) { + continue; + } + final var sourceInterval = sourceIntervalOptional.get(); + int index = lowerBoundByStart(sourceFeatures, sourceInterval.start()); + if (index > 0) { + index--; + } + for (int i = index; i < sourceFeatures.size(); i++) { + final var feature = sourceFeatures.get(i); + if (feature.start() >= sourceInterval.end()) { + break; + } + if (feature.end() <= sourceInterval.start()) { + continue; + } + projectSourceIntervalOnSegment( + segment, + feature.start(), + feature.end(), + feature.value(), + feature.label(), + queryStartPx, + queryEndPx, + bpResolution + ).ifPresent(consumer); + } + } + } + } + private static int lowerBoundByStart(final @NotNull List features, final long targetStart) { int lo = 0; int hi = features.size(); @@ -2044,8 +2344,11 @@ private record QueryPxRange(long startPx, long endPx) { private record PrecomputeTask(long bpResolution, @NotNull String modeKey, long totalVisiblePixels, - @NotNull String assemblySignature, - @NotNull Path sidecarFile) { + @NotNull String assemblySignature) { + } + + private record ComputedPrecomputeTask(@NotNull PrecomputeTask task, + @NotNull PrecomputedSeries series) { } private record PrecomputedSeriesKey(@NotNull String trackId, diff --git a/version.txt b/version.txt index 47207a6..abff7f1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.0.86-ad8acb7-webui_543dcbb \ No newline at end of file +1.0.89-8494e37-webui_543dcbb \ No newline at end of file