Fix: audio timing under timelapses

src/main/java/com/moulberry/flashback/editor/ui/windows/TimelineWindow.java

@@ -28,6 +28,7 @@
 import com.moulberry.flashback.editor.ui.ImGuiHelper;
 import com.moulberry.flashback.record.FlashbackMeta;
 import com.moulberry.flashback.state.KeyframeTrack;
+import com.moulberry.flashback.state.RealTimeMapping;
 import imgui.moulberry90.ImDrawList;
 import imgui.moulberry90.ImGui;
 import imgui.moulberry90.ImVec4;
@@ -167,12 +168,13 @@ public static void render() {
         if (timelineVisible) {
             FlashbackMeta metadata = replayServer.getMetadata();
             editorState = EditorStateManager.get(metadata.replayIdentifier);
+            RealTimeMapping realTimeMapping = editorState.getRealTimeMapping();
 
             editorSceneStamp = editorState.acquireRead();
             editorSceneStampIsWrite = false;
             try {
                 editorScene = editorState.getCurrentScene(editorSceneStamp);
-                renderInner(replayServer, metadata);
+                renderInner(replayServer, metadata, realTimeMapping);
             } finally {
                 editorState.release(editorSceneStamp);
                 editorSceneStamp = 0L;
@@ -183,7 +185,7 @@ public static void render() {
         ImGui.end();
     }
 
-    private static void renderInner(ReplayServer replayServer, FlashbackMeta metadata) {
+    private static void renderInner(ReplayServer replayServer, FlashbackMeta metadata, @Nullable RealTimeMapping realTimeMapping) {
         ImDrawList drawList = ImGui.getWindowDrawList();
 
         float maxX = ImGui.getWindowContentRegionMaxX();
@@ -336,7 +338,7 @@ private static void renderInner(ReplayServer replayServer, FlashbackMeta metadat
         renderKeyframeElements(x, contentY, cursorTicks, middleX);
 
         childDrawList.pushClipRect(x + middleX + 1, y + middleY, x + width, y + height);
-        renderKeyframes(x, contentY, mouseX, minTicks, availableTicks, totalTicks);
+        renderKeyframes(x, contentY, mouseX, minTicks, availableTicks, totalTicks, realTimeMapping);
         childDrawList.popClipRect();
 
         ImGui.endChild();
@@ -550,7 +552,7 @@ private static void renderInner(ReplayServer replayServer, FlashbackMeta metadat
                     }
                 }
             } else if (mouseX < x + width - 2 && (leftClicked || rightClicked)) {
-                handleClick(replayServer, totalTicks, contentY);
+                handleClick(replayServer, totalTicks, contentY, realTimeMapping);
                 if (leftClicked) {
                     draggingMouseButton = ImGuiMouseButton.Left;
                 } else {
@@ -924,7 +926,7 @@ private static void removeAllSelectedKeyframes() {
         editorState.markDirty();
     }
 
-    private static void handleClick(ReplayServer replayServer, int totalTicks, float contentY) {
+    private static void handleClick(ReplayServer replayServer, int totalTicks, float contentY, @Nullable RealTimeMapping realTimeMapping) {
         releaseGrabbed(replayServer, totalTicks, contentY);
         List<SelectedKeyframes> oldSelectedKeyframesList = new ArrayList<>(selectedKeyframesList);
         selectedKeyframesList.clear();
@@ -1066,7 +1068,7 @@ private static void handleClick(ReplayServer replayServer, int totalTicks, float
                 Map.Entry<Integer, Keyframe> closest = null;
 
                 Map.Entry<Integer, Keyframe> floor = keyframeTrack.keyframesByTick.floorEntry(tick);
-                float floorCustomWidth = floor == null ? -1 : floor.getValue().getCustomWidthInTicks();
+                float floorCustomWidth = floor == null ? -1 : floor.getValue().getCustomWidthInTicks(realTimeMapping, floor.getKey());
 
                 if (floor != null && floorCustomWidth > 0) {
                     int tickMax = floor.getKey() + (int) Math.ceil(floorCustomWidth);
@@ -1589,7 +1591,8 @@ private static GrabMovementInfo calculateGrabMovementInfo(int totalTicks) {
         return new GrabMovementInfo(grabbedDelta, grabbedScalePivotTick, grabbedScaleFactor);
     }
 
-    private static void renderKeyframes(float x, float y, float mouseX, int minTicks, float availableTicks, int totalTicks) {
+    private static void renderKeyframes(float x, float y, float mouseX, int minTicks, float availableTicks, int totalTicks,
+                                        @Nullable RealTimeMapping realTimeMapping) {
         float lineHeight = ImGui.getTextLineHeightWithSpacing() + ImGui.getStyle().getItemSpacingY();
 
         ImDrawList drawList = ImGui.getWindowDrawList();
@@ -1652,7 +1655,7 @@ private static void renderKeyframes(float x, float y, float mouseX, int minTicks
                     float midX = x + keyframeX;
 
                     keyframe.drawOnTimeline(drawList, keyframeSize, midX, midY, keyframeTrack.enabled ? 0xFF0000FF : 0x800000FF,
-                            timelineScale, minTimelineX, maxTimelineX, tick, keyframeTimes);
+                            timelineScale, minTimelineX, maxTimelineX, realTimeMapping, tick, keyframeTimes);
                 } else {
                     int keyframeX = replayTickToTimelineX(tick);
 
@@ -1685,7 +1688,7 @@ private static void renderKeyframes(float x, float y, float mouseX, int minTicks
                     }
 
                     keyframe.drawOnTimeline(drawList, keyframeSize, midX, midY, colour,
-                            timelineScale, minTimelineX, maxTimelineX, tick, keyframeTimes);
+                            timelineScale, minTimelineX, maxTimelineX, realTimeMapping, tick, keyframeTimes);
                 }
 
                 if ((selectedKeyframesForTrack == null || grabMovementInfo == null) && keyframeTrack.keyframeType == TimelapseKeyframeType.INSTANCE) {
@@ -2265,3 +2268,4 @@ private static String ticksToTimestamp(int ticks) {
     }
 
 }
+

src/main/java/com/moulberry/flashback/keyframe/Keyframe.java

@@ -4,6 +4,7 @@
 import com.moulberry.flashback.keyframe.change.KeyframeChange;
 import com.moulberry.flashback.keyframe.impl.*;
 import com.moulberry.flashback.keyframe.interpolation.InterpolationType;
+import com.moulberry.flashback.state.RealTimeMapping;
 import imgui.moulberry90.ImDrawList;
 import org.jetbrains.annotations.Nullable;
 
@@ -32,13 +33,17 @@ public void interpolationType(InterpolationType interpolationType) {
     public abstract @Nullable KeyframeChange createHermiteInterpolatedChange(Map<Float, Keyframe> keyframes, float tick);
 
     public float getCustomWidthInTicks() {
+        return this.getCustomWidthInTicks(null, 0);
+    }
+
+    public float getCustomWidthInTicks(@Nullable RealTimeMapping realTimeMapping, int tick) {
         return -1;
     }
 
     public void renderEditKeyframe(Consumer<Consumer<Keyframe>> update) {}
 
     public void drawOnTimeline(ImDrawList drawList, int keyframeSize, float x, float y, int colour, float timelineScale, float minTimelineX, float maxTimelineX,
-                               int tick, TreeMap<Integer, Keyframe> keyframeTimes) {
+                               @Nullable RealTimeMapping realTimeMapping, int tick, TreeMap<Integer, Keyframe> keyframeTimes) {
         int easeSize = keyframeSize / 5;
         switch (interpolationType) {
             case SMOOTH -> {
@@ -162,3 +167,5 @@ public JsonElement serialize(Keyframe src, Type typeOfSrc, JsonSerializationCont
     }
 
 }
+
+

src/main/java/com/moulberry/flashback/keyframe/KeyframeType.java

@@ -2,6 +2,7 @@
 
 import com.moulberry.flashback.keyframe.change.KeyframeChange;
 import com.moulberry.flashback.keyframe.handler.KeyframeHandler;
+import com.moulberry.flashback.state.RealTimeMapping;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.TreeMap;
@@ -43,7 +44,7 @@ default boolean cullKeyframesInTimelineToTheLeft() {
     default boolean hasCustomKeyframeChangeCalculation() {
         return false;
     }
-    default KeyframeChange customKeyframeChange(TreeMap<Integer, Keyframe> keyframes, float tick) {
+    default KeyframeChange customKeyframeChange(TreeMap<Integer, Keyframe> keyframes, float tick, @Nullable RealTimeMapping realTimeMapping) {
         throw new UnsupportedOperationException();
     }
 

src/main/java/com/moulberry/flashback/keyframe/change/KeyframeChangePlayAudio.java

@@ -21,10 +21,8 @@ public KeyframeChangePlayAudio(FlashbackAudioBuffer audioBuffer, int startTick,
     public void apply(KeyframeHandler keyframeHandler) {
         Minecraft minecraft = keyframeHandler.getMinecraft();
         if (minecraft != null && minecraft.level != null) {
-            var tickRateManager = minecraft.level.tickRateManager();
-            float tickrate = tickRateManager.tickrate();
             FlashbackAudioManager.playAt(minecraft.getSoundManager().soundEngine, this.audioBuffer, this.startTick,
-                    this.seconds, tickrate / 20f);
+                    this.seconds, 1.0f);
         }
     }
 

src/main/java/com/moulberry/flashback/keyframe/impl/AudioKeyframe.java

@@ -9,6 +9,7 @@
 import com.moulberry.flashback.keyframe.types.AudioKeyframeType;
 import com.moulberry.flashback.sound.FlashbackAudioBuffer;
 import com.moulberry.flashback.sound.FlashbackAudioManager;
+import com.moulberry.flashback.state.RealTimeMapping;
 import imgui.moulberry90.ImDrawList;
 import org.jetbrains.annotations.Nullable;
 
@@ -79,18 +80,31 @@ private void ensureAudioBufferLoaded() {
         return new KeyframeChangePlayAudio(this.audioBuffer, startTick, seconds);
     }
 
-    @Override
-    public float getCustomWidthInTicks() {
+    private float getDurationInTicks(@Nullable RealTimeMapping mapping, int tick) {
         this.ensureAudioBufferLoaded();
         if (this.audioBuffer == FlashbackAudioBuffer.EMPTY) {
             return -1;
         }
-        return this.audioBuffer.durationInSeconds() * 20.0f;
+
+        float durationInTicks = this.audioBuffer.durationInSeconds() * 20.0f;
+        if (mapping == null) {
+            return durationInTicks;
+        }
+
+        float startRealTime = mapping.getRealTime(tick);
+        float endRealTime = startRealTime + durationInTicks;
+        return mapping.getTickForRealTime(endRealTime) - tick;
+    }
+
+    @Override
+    public float getCustomWidthInTicks(@Nullable RealTimeMapping mapping, int tick) {
+        return this.getDurationInTicks(mapping, tick);
     }
 
     @Override
     public void drawOnTimeline(ImDrawList drawList, int keyframeSize, float x, float y, int colour,
-                               float timelineScale, float minTimelineX, float maxTimelineX, int tick, TreeMap<Integer, Keyframe> keyframeTimes) {
+                               float timelineScale, float minTimelineX, float maxTimelineX, @Nullable RealTimeMapping realTimeMapping,
+                               int tick, TreeMap<Integer, Keyframe> keyframeTimes) {
         this.ensureAudioBufferLoaded();
 
         int alpha = colour & 0xFF000000;
@@ -101,7 +115,8 @@ public void drawOnTimeline(ImDrawList drawList, int keyframeSize, float x, float
             return;
         }
 
-        float durationInTicks = this.audioBuffer.durationInSeconds() * 20.0f;
+        float durationInTicks = this.getDurationInTicks(realTimeMapping, tick);
+
         int waveformLength = (int)(durationInTicks / timelineScale);
         int drawLength = waveformLength;
 
@@ -141,3 +156,5 @@ public JsonElement serialize(AudioKeyframe src, Type typeOfSrc, JsonSerializatio
         }
     }
 }
+
+

src/main/java/com/moulberry/flashback/keyframe/types/AudioKeyframeType.java

@@ -8,6 +8,7 @@
 import com.moulberry.flashback.keyframe.handler.KeyframeHandler;
 import com.moulberry.flashback.keyframe.handler.MinecraftKeyframeHandler;
 import com.moulberry.flashback.keyframe.impl.AudioKeyframe;
+import com.moulberry.flashback.state.RealTimeMapping;
 import imgui.moulberry90.ImGui;
 import net.fabricmc.loader.api.FabricLoader;
 import net.minecraft.client.resources.language.I18n;
@@ -71,16 +72,21 @@ public boolean hasCustomKeyframeChangeCalculation() {
     }
 
     @Override
-    public KeyframeChange customKeyframeChange(TreeMap<Integer, Keyframe> keyframes, float tick) {
+    public KeyframeChange customKeyframeChange(TreeMap<Integer, Keyframe> keyframes, float tick, @Nullable RealTimeMapping realTimeMapping) {
         Map.Entry<Integer, Keyframe> entry = keyframes.floorEntry((int) tick);
         if (entry == null) {
             return null;
         }
 
-        float delta = tick - entry.getKey();
+        float seconds;
+        if (realTimeMapping != null) {
+            seconds = (realTimeMapping.getRealTime(tick) - realTimeMapping.getRealTime(entry.getKey())) / 20.0f;
+        } else {
+            seconds = (tick - entry.getKey()) / 20.0f;
+        }
 
         AudioKeyframe audioKeyframe = (AudioKeyframe) entry.getValue();
-        return audioKeyframe.createAudioChange(entry.getKey(), delta / 20.0f);
+        return audioKeyframe.createAudioChange(entry.getKey(), seconds);
     }
 
     @Override

src/main/java/com/moulberry/flashback/state/EditorState.java

@@ -145,6 +145,11 @@ public Camera getAudioCamera() {
         return dummyCamera;
     }
 
+    public @Nullable RealTimeMapping getRealTimeMapping() {
+        updateRealtimeMappingsIfNeeded();
+        return this.realTimeMapping;
+    }
+
     public void markDirty() {
         this.dirty = true;
         this.modCount += 1;
@@ -201,6 +206,9 @@ public void applyKeyframes(KeyframeHandler keyframeHandler, float tick) {
 
         updateRealtimeMappingsIfNeeded();
 
+        FlashbackConfigV1 config = Flashback.getConfig();
+        RealTimeMapping interpolationMapping = config.keyframes.useRealtimeInterpolation ? this.realTimeMapping : null;
+
         long stamp = this.sceneLock.readLock();
         try {
             for (KeyframeTrack keyframeTrack : this.currentScene().keyframeTracks) {
@@ -222,7 +230,7 @@ public void applyKeyframes(KeyframeHandler keyframeHandler, float tick) {
 
                 // Try to apply keyframes, mark applied if successful
 
-                KeyframeChange change = keyframeTrack.createKeyframeChange(tick, this.realTimeMapping);
+                KeyframeChange change = keyframeTrack.createKeyframeChange(tick, interpolationMapping, this.realTimeMapping);
                 if (change == null) {
                     if (keyframeHandler.alwaysApplyLastKeyframe() && !keyframeTrack.keyframeType.neverApplyLastKeyframe() && !keyframeTrack.keyframesByTick.isEmpty()) {
                         if (keyframeTrack.keyframesByTick.lastKey() <= tick) {
@@ -247,7 +255,7 @@ public void applyKeyframes(KeyframeHandler keyframeHandler, float tick) {
             if (keyframeHandler.alwaysApplyLastKeyframe() && !maybeApplyLastTick.isEmpty()) {
                 for (Map.Entry<Class<? extends KeyframeChange>, KeyframeTrack> entry : maybeApplyLastTick.entrySet()) {
                     KeyframeTrack keyframeTrack = entry.getValue();
-                    KeyframeChange change = keyframeTrack.createKeyframeChange(keyframeTrack.keyframesByTick.lastKey(), this.realTimeMapping);
+                    KeyframeChange change = keyframeTrack.createKeyframeChange(keyframeTrack.keyframesByTick.lastKey(), interpolationMapping, this.realTimeMapping);
 
                     if (change == null) {
                         continue;
@@ -268,14 +276,7 @@ public void applyKeyframes(KeyframeHandler keyframeHandler, float tick) {
     private void updateRealtimeMappingsIfNeeded() {
         long stamp = this.sceneLock.readLock();
         try {
-            FlashbackConfigV1 config = Flashback.getConfig();
-            if (!config.keyframes.useRealtimeInterpolation) {
-                this.sceneLock.unlock(stamp);
-                stamp = this.sceneLock.writeLock();
-
-                this.lastRealTimeMappingModCount = this.modCount;
-                this.realTimeMapping = null;
-            } else if (this.realTimeMapping == null || this.lastRealTimeMappingModCount != this.modCount) {
+            if (this.realTimeMapping == null || this.lastRealTimeMappingModCount != this.modCount) {
                 this.sceneLock.unlock(stamp);
                 stamp = this.sceneLock.writeLock();
 
@@ -332,9 +333,10 @@ private void calculateRealtimeMappings() {
             return;
         }
 
-        float lastSpeed = Float.NaN;
+        float lastSpeed = 1.0f;
 
         for (int tick = start; tick <= end; tick++) {
+            boolean foundTickrate = false;
             for (KeyframeTrack keyframeTrack : applicableTracks) {
                 KeyframeChange change = keyframeTrack.createKeyframeChange(tick, this.realTimeMapping);
                 if (!(change instanceof KeyframeChangeTickrate changeTickrate)) {
@@ -346,8 +348,13 @@ private void calculateRealtimeMappings() {
                     lastSpeed = newSpeed;
                     this.realTimeMapping.addMapping(tick, newSpeed);
                 }
+                foundTickrate = true;
                 break;
             }
+            if (!foundTickrate && lastSpeed != 1.0f) {
+                lastSpeed = 1.0f;
+                this.realTimeMapping.addMapping(tick, 1.0f);
+            }
         }
 
         // Check if the tick afterwards has a change, if it does then the last keyframe is probably a hold keyframe
@@ -431,6 +438,7 @@ public StartAndEnd getExportStartAndEnd() {
     public StartAndEnd getFirstAndLastTicksInTracks() {
         int start = -1;
         int end = -1;
+        RealTimeMapping realTimeMapping = this.getRealTimeMapping();
 
         long stamp = this.sceneLock.readLock();
         try {
@@ -442,7 +450,7 @@ public StartAndEnd getFirstAndLastTicksInTracks() {
                 var entry = keyframeTrack.keyframesByTick.lastEntry();
 
                 int max = entry.getKey();
-                float lastCustomWidth = entry.getValue().getCustomWidthInTicks();
+                float lastCustomWidth = entry.getValue().getCustomWidthInTicks(realTimeMapping, entry.getKey());
                 if (lastCustomWidth > 0) {
                     max = entry.getKey() + (int) Math.ceil(lastCustomWidth);
                 }
@@ -466,3 +474,5 @@ public StartAndEnd getFirstAndLastTicksInTracks() {
     }
 
 }
+
+

src/main/java/com/moulberry/flashback/state/KeyframeTrack.java

@@ -37,8 +37,13 @@ public KeyframeTrack(KeyframeType<?> keyframeType) {
 
     @Nullable
     public KeyframeChange createKeyframeChange(float tick, @Nullable RealTimeMapping realTimeMapping) {
+        return createKeyframeChange(tick, realTimeMapping, realTimeMapping);
+    }
+
+    @Nullable
+    public KeyframeChange createKeyframeChange(float tick, @Nullable RealTimeMapping realTimeMapping, @Nullable RealTimeMapping customRealTimeMapping) {
         if (this.keyframeType.hasCustomKeyframeChangeCalculation()) {
-            return this.keyframeType.customKeyframeChange(this.keyframesByTick, tick);
+            return this.keyframeType.customKeyframeChange(this.keyframesByTick, tick, customRealTimeMapping);
         }
         if (this.keyframeType == TimelapseKeyframeType.INSTANCE) {
             return this.tryApplyKeyframesTimelapse(tick);