Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@
import com.opentest.service.dashboard.framework.ProjectTriggerFacts;
import com.opentest.service.dashboard.framework.SavedMeasurementInputFacts;
import com.opentest.service.export.waveform.source.FrozenWaveformSource;
import com.opentest.service.export.waveform.source.FrozenWaveformShard;
import com.opentest.service.export.waveform.source.WaveformSourceType;
import com.opentest.service.export.waveform.source.WaveformSourceReader;
import com.opentest.service.export.waveform.source.WaveformSourceReaderFactory;
import com.opentest.service.grpc.provider.ProviderExecutionInputSnapshots;
Expand All @@ -56,8 +58,11 @@
import com.opentest.service.virtualchannel.VirtualChannelExecutionChannelResolver;
import jakarta.annotation.Resource;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
Expand Down Expand Up @@ -233,6 +238,8 @@ private List<File> copyInputs(ReadPlan saved, FlowExecuteSnapshotDataDto target,
if (Files.size(root.resolve(path)) != shard.getExpectedBytes()) throw new IOException("Incomplete saved input copy");
names.add(path.toString().replace('\\', '/'));
}
if (frozen.getSourceType() == WaveformSourceType.WAV)
writeChildWavIndex(frozen, names.stream().map(root::resolve).toList(), cancelled);
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
row.setFileName(names.getFirst()); row.setFileUrl(names.getFirst()); row.setShardsNo(names.size());
copied.add(row); byId.put(source.getId(), row); shardNames.put(source.getId(), names);
}
Expand All @@ -252,6 +259,67 @@ private List<File> copyInputs(ReadPlan saved, FlowExecuteSnapshotDataDto target,
return List.copyOf(copied);
}

private void writeChildWavIndex(FrozenWaveformSource source, List<Path> copies,
BooleanSupplier cancelled) throws IOException {
var original = wavIndexPath(source.getShards().getFirst().getPath());
var entries = new ArrayList<String>();
if (Files.exists(original, LinkOption.NOFOLLOW_LINKS)) {
if (!Files.isRegularFile(original)) throw new IllegalArgumentException("Saved WAV index is unavailable");
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
var lines = Files.readAllLines(original, StandardCharsets.UTF_8);
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
if (lines.size() != copies.size()) throw new IllegalArgumentException("Saved WAV index does not match its frozen shards");
double end = 0;
long frames = 0;
for (int index = 0; index < lines.size(); index++) {
checkCancelled(cancelled);
var fields = lines.get(index).split("\\|", -1);
if (fields.length < 2 || fields.length > 3 || fields[0].isBlank())
throw new IllegalArgumentException("Invalid saved WAV index entry");
var indexed = Path.of(fields[0].replace('\\', '/'));
if (!indexed.isAbsolute()) indexed = original.getParent().resolve(indexed);
if (!indexed.toAbsolutePath().normalize().equals(source.getShards().get(index).getPath().toAbsolutePath().normalize()))
throw new IllegalArgumentException("Saved WAV index names a different frozen shard");
double nextEnd = Double.parseDouble(fields[1]);
if (!Double.isFinite(nextEnd) || nextEnd < end)
throw new IllegalArgumentException("Saved WAV index time is invalid or regressing");
end = nextEnd;
if (fields.length == 3) {
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
long nextFrames = Long.parseLong(fields[2].trim());
if (nextFrames < frames) throw new IllegalArgumentException("Saved WAV index frame count is regressing");
frames = nextFrames;
}
entries.add(copies.get(index).toAbsolutePath().normalize().toString().replace('\\', '/')
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
+ "|" + fields[1] + (fields.length == 3 ? "|" + fields[2] : ""));
}
} else {
long samples = 0;
for (var copy : copies) {
checkCancelled(cancelled);
var copied = new FrozenWaveformSource(source.getSourceFileId(), WaveformSourceType.WAV,
source.getFrozenSamplingRate(), List.of(new FrozenWaveformShard(copy, Files.size(copy))),
source.getFrozenBitDepth(), source.getFrozenFloatingPoint());
try (var reader = readers.open(copied)) {
while (true) {
checkCancelled(cancelled);
int count = reader.read(8192).getFrameCount();
if (count == 0) break;
samples = Math.addExact(samples, count);
}
}
entries.add(copy.toAbsolutePath().normalize().toString().replace('\\', '/')
+ "|" + samples * 1000D / source.getFrozenSamplingRate());
}
}
checkCancelled(cancelled);
Files.write(wavIndexPath(copies.getFirst()), entries, StandardCharsets.UTF_8,
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
}

private static Path wavIndexPath(Path firstShard) {
String name = firstShard.getFileName().toString();
String stem = name.substring(0, name.lastIndexOf('.')).replaceFirst("_\\d+(?:_\\d{14})$", "");
Comment thread
mingtaojin marked this conversation as resolved.
Resolved
return firstShard.resolveSibling(stem + ".idx");
}

private Map<Class<?>, Object> services(ReadPlan source, FlowExecuteSnapshotDataDto target,
Map<String, Object> configuration, MeasureReanalysisOutputPlan plan,
MeasureReanalysisOutputs output, BoundExecutionGates gates) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ synchronized List<File> finish() {
row.setType(FileTypeEnum.CRYBIN.getType()); row.setShardsNo(shards.size());
String filename = Path.of(name).getFileName().toString();
row.setBusinessId(filename.substring(0, filename.length() - ".crybin".length()));
row.setFileName(shards.getFirst()); row.setFileUrl(root.resolve(shards.getFirst()).normalize().toString()); rows.add(row);
row.setFileName(shards.getFirst()); row.setFileUrl(shards.getFirst()); rows.add(row);
}
writer.close(); active(); state = State.FINISHED;
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import com.opentest.dashboard.api.unit.GraphicUnitResolution;
import com.opentest.service.FileService;
import com.opentest.service.VariablesService;
import com.opentest.service.component.ProjectDashboardComponent;
import com.opentest.service.dashboard.framework.DashboardOperationRegistry;
import com.opentest.service.dashboard.framework.MeasurementInputInterpreter;
import com.opentest.service.dashboard.framework.PluginDashboardConfigurationResolver;
Expand Down Expand Up @@ -137,6 +138,112 @@ class MeasureReanalysisExecutorTest {
private static final RecordedCallScope CALLER = new RecordedCallScope(1, 1, "session", Locale.ROOT);
@TempDir Path directory;

@Test void publishesAChildOwnedWavIndexWithTheOriginalTimingAndLogicalFrameCount() throws Exception {
var fixture = fixture();
Files.writeString(directory.resolve("raw.idx"), fixture.raw.toString().replace('\\', '/') + "|341.4791666666667|7\n");
byte[] originalIndex = Files.readAllBytes(directory.resolve("raw.idx"));
var source = fixture.history(8);
var destination = fixture.destination(source, 9);

fixture.executor.execute(source, destination, CALLER, fixture.cancelled::get);

var copied = fixture.onlyFile(9, FileTypeEnum.WAV);
var rows = Files.readAllLines(fixture.indexPath(copied));
assertEquals(List.of(fixture.path(copied).toString().replace('\\', '/') + "|341.4791666666667|7"), rows);
assertArrayEquals(originalIndex, Files.readAllBytes(directory.resolve("raw.idx")));
assertEquals(source.getSnapshotData(), fixture.history(8).getSnapshotData());
assertArrayEquals(expectedSamples(1), fixture.read(copied), 1e-12);
}

@Test void rebindsTimestampedWavShardsInOrderWithoutInventingLogicalFrameCounts() throws Exception {
var fixture = fixture();
var source = fixture.timestampedShards();
byte[] originalIndex = Files.readAllBytes(directory.resolve("Measure #3_Ch1_Acquired Waveform.idx"));
var destination = fixture.destination(source, 9);

fixture.executor.execute(source, destination, CALLER, fixture.cancelled::get);

var copied = fixture.onlyFile(9, FileTypeEnum.WAV);
assertEquals(2, copied.getShardsNo());
Path parent = fixture.path(copied).getParent();
Path first = parent.resolve("Measure #3_Ch1_Acquired Waveform_1_20261113160955.wav");
Path second = parent.resolve("Measure #3_Ch1_Acquired Waveform_2_20261113160956.wav");
assertEquals(List.of(first.toString().replace('\\', '/') + "|170.66666666666666|3",
second.toString().replace('\\', '/') + "|341.4791666666667|7"),
Files.readAllLines(fixture.indexPath(copied)));
assertTrue(Files.isRegularFile(first)); assertTrue(Files.isRegularFile(second));
assertArrayEquals(originalIndex, Files.readAllBytes(directory.resolve("Measure #3_Ch1_Acquired Waveform.idx")));
fixture.assertCompleteInput();
}

@ParameterizedTest
@ValueSource(strings = {"empty", "wrong-shard", "invalid-time", "invalid-frames", "partial", "reversed"})
void invalidInputIndexesNeverPublishAChildOrMutateOriginals(String damage) throws Exception {
var fixture = fixture();
var source = fixture.timestampedShards();
Path index = directory.resolve("Measure #3_Ch1_Acquired Waveform.idx");
var lines = Files.readAllLines(index);
switch (damage) {
case "empty" -> Files.writeString(index, "");
case "wrong-shard" -> Files.writeString(index, directory.resolve("unrelated.wav") + "|1|1\n" + lines.getLast() + "\n");
case "invalid-time" -> Files.writeString(index, lines.getFirst() + "\n" + lines.getLast().replace("341.4791666666667", "NaN") + "\n");
case "invalid-frames" -> Files.writeString(index, lines.getFirst() + "\n" + lines.getLast().replace("|7", "|2") + "\n");
case "partial" -> Files.writeString(index, lines.getFirst() + "\n");
case "reversed" -> Files.writeString(index, lines.getLast() + "\n" + lines.getFirst() + "\n");
default -> throw new AssertionError(damage);
}
String originalIndex = Files.exists(index) ? Files.readString(index) : null;
var destination = fixture.destination(source, 9);
String draft = destination.getSnapshotData();

assertThrows(IllegalArgumentException.class,
() -> fixture.executor.execute(source, destination, CALLER, fixture.cancelled::get));

fixture.assertUnpublished(9, draft);
assertFalse(Files.exists(directory.resolve("1/9")));
assertEquals(source.getSnapshotData(), fixture.history(8).getSnapshotData());
assertEquals(originalIndex, Files.exists(index) ? Files.readString(index) : null);
verify(fixture.plugins, never()).bindMeasurement(anyString(), anyMap(), anyMap(), any(), any(), any());
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
void indexlessSavedWavsRemainRecalculableWithTruthfulDurationAndNoInventedFrameCount(boolean multishard) throws Exception {
var fixture = fixture();
var source = multishard ? fixture.timestampedShards() : fixture.history(8);
if (multishard) Files.delete(directory.resolve("Measure #3_Ch1_Acquired Waveform.idx"));
var destination = fixture.destination(source, 9);

fixture.executor.execute(source, destination, CALLER, fixture.cancelled::get);

var copied = fixture.onlyFile(9, FileTypeEnum.WAV);
var lines = Files.readAllLines(fixture.indexPath(copied));
assertEquals(multishard ? 2 : 1, lines.size());
double lastEnd = 0;
for (String line : lines) {
var fields = line.split("\\|", -1);
assertEquals(2, fields.length, "Absent original append-frame counts must remain absent");
Path copiedShard = Path.of(fields[0]);
assertTrue(copiedShard.startsWith(fixture.path(copied).getParent()));
assertTrue(Files.isRegularFile(copiedShard));
assertTrue(Double.parseDouble(fields[1]) >= lastEnd);
lastEnd = Double.parseDouble(fields[1]);
}
assertEquals(SAMPLES * 1000D / RATE, lastEnd, 1e-9);
fixture.assertCompleteInput();
assertEquals(source.getSnapshotData(), fixture.history(8).getSnapshotData());
}

@Test void preservesLegacyTwoColumnIndexTimingWithoutAddingLogicalFrameCounts() throws Exception {
var fixture = fixture();
Files.writeString(directory.resolve("raw.idx"), fixture.raw.toString().replace('\\', '/') + "|999.125\n");
var source = fixture.history(8);
fixture.executor.execute(source, fixture.destination(source, 9), CALLER, fixture.cancelled::get);
var copied = fixture.onlyFile(9, FileTypeEnum.WAV);
assertEquals(List.of(fixture.path(copied).toString().replace('\\', '/') + "|999.125"),
Files.readAllLines(fixture.indexPath(copied)));
}

@Test void readsTheFullOriginalThroughItsLastSampleAndPublishesIndependentFiles() throws Exception {
var fixture = fixture();
var source = fixture.history(8);
Expand Down Expand Up @@ -609,6 +716,34 @@ void savedStages() throws IOException {
var source = history(8); source.setSnapshotData(snapshot.toJson()); histories.updateById(source);
}

FlowExecuteResult timestampedShards() throws IOException {
Files.delete(raw);
List<String> names = List.of("Measure #3_Ch1_Acquired Waveform_1_20261113160955.wav",
"Measure #3_Ch1_Acquired Waveform_2_20261113160956.wav");
Files.write(directory.resolve(names.getFirst()), wav(0, 8192));
Files.write(directory.resolve(names.getLast()), wav(8192, SAMPLES - 8192));
Files.writeString(directory.resolve("Measure #3_Ch1_Acquired Waveform.idx"),
directory.resolve(names.getFirst()).toString().replace('\\', '/') + "|170.66666666666666|3\n"
+ directory.resolve(names.getLast()).toString().replace('\\', '/') + "|341.4791666666667|7\n");
var row = fileRows.selectById(10L); row.setFileName(names.getFirst()); row.setFileUrl(names.getFirst());
row.setShardsNo(2); fileRows.updateById(row);
var source = history(8);
var snapshot = FlowExecuteSnapshotDataDto.parseJson(source.getSnapshotData());
var channels = snapshot.getDashboardConfig().getDashboardItemConfig().getCollectConfig().getChannels();
var plan = RecordedWaveformPlans.selected(channels);
RecordedWaveformPlans.close(plan, List.of(new RawInputArtifact(CHANNEL, 8, "raw", names,
RATE, 100L, FIRST_SAMPLE, SAMPLES, true)));
snapshot.setRecordedWaveformPlan(plan); source.setSnapshotData(snapshot.toJson());
histories.updateById(source);
return source;
}

Path indexPath(File row) {
String relative = ReflectionTestUtils.invokeMethod(new ProjectDashboardComponent(),
"resolveWavIndexRelativePath", row.getFileUrl(), 1L, row.getRequestId());
return directory.resolve(relative);
}

FlowExecuteResult history(long id) { return histories.selectById(id); }

FlowExecuteResult destination(FlowExecuteResult source, long id) {
Expand Down Expand Up @@ -823,14 +958,18 @@ private static byte[] stageWav(String phase) {
}

private static byte[] wav(int samples) {
return wav(0, samples);
}

private static byte[] wav(int offset, int samples) {
int dataBytes = samples * 2;
var buffer = ByteBuffer.allocate(44 + dataBytes).order(ByteOrder.LITTLE_ENDIAN);
buffer.put("RIFF".getBytes(StandardCharsets.US_ASCII)).putInt(36 + dataBytes)
.put("WAVEfmt ".getBytes(StandardCharsets.US_ASCII)).putInt(16)
.putShort((short) 1).putShort((short) 1).putInt(RATE).putInt(RATE * 2)
.putShort((short) 2).putShort((short) 16)
.put("data".getBytes(StandardCharsets.US_ASCII)).putInt(dataBytes);
for (int i = 0; i < samples; i++) buffer.putShort(sample(i));
for (int i = 0; i < samples; i++) buffer.putShort(sample(offset + i));
return buffer.array();
}

Expand Down
Loading
Loading