From fce79d326c72ab12c24840a7d3c83644685450d9 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:44 -0300 Subject: [PATCH 001/464] feat(base): add a bounded blocking queue with close and conditional pop --- libs/base/include/oma/base/bounded_queue.hpp | 93 +++++++++++++++++ tests/base/main.cpp | 2 + tests/base/test_bounded_queue.cpp | 101 +++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 libs/base/include/oma/base/bounded_queue.hpp create mode 100644 tests/base/test_bounded_queue.cpp diff --git a/libs/base/include/oma/base/bounded_queue.hpp b/libs/base/include/oma/base/bounded_queue.hpp new file mode 100644 index 0000000..33fdd0a --- /dev/null +++ b/libs/base/include/oma/base/bounded_queue.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +// A bounded FIFO between pipeline stages (CLAUDE.md §13: queues between stages are bounded, +// giving backpressure). The producer blocks while the queue is full; the consumer never blocks. +// close() releases a blocked producer for cancellation and shutdown. +// +// Not for the real-time audio thread: it takes a mutex. Real-time paths use lock-free rings. + +namespace oma { + +template +class BoundedQueue { +public: + explicit BoundedQueue(std::size_t capacity) : capacity_(capacity == 0 ? 1 : capacity) {} + + BoundedQueue(const BoundedQueue&) = delete; + BoundedQueue& operator=(const BoundedQueue&) = delete; + BoundedQueue(BoundedQueue&&) = delete; + BoundedQueue& operator=(BoundedQueue&&) = delete; + ~BoundedQueue() = default; + + // Waits for room, then appends. Returns false (dropping `value`) once the queue is closed. + bool push(T value) { + std::unique_lock lock(mutex_); + room_.wait(lock, [&] { return closed_ || items_.size() < capacity_; }); + if (closed_) { + return false; + } + items_.push_back(std::move(value)); + return true; + } + + // The oldest item, if any. + std::optional try_pop() { + return try_pop_if([](const T&) { return true; }); + } + + // The oldest item, only if `take(item)` says so; otherwise it stays queued. + template + std::optional try_pop_if(Pred take) { + std::optional out; + { + const std::scoped_lock lock(mutex_); + if (items_.empty() || !take(std::as_const(items_.front()))) { + return out; + } + out.emplace(std::move(items_.front())); + items_.pop_front(); + } + room_.notify_one(); + return out; + } + + // Drops everything queued (the producer may continue). + void clear() { + { + const std::scoped_lock lock(mutex_); + items_.clear(); + } + room_.notify_all(); + } + + // Wakes a blocked producer; every later push fails. Queued items stay readable. + void close() { + { + const std::scoped_lock lock(mutex_); + closed_ = true; + } + room_.notify_all(); + } + + [[nodiscard]] std::size_t size() const { + const std::scoped_lock lock(mutex_); + return items_.size(); + } + [[nodiscard]] std::size_t capacity() const noexcept { return capacity_; } + +private: + std::size_t capacity_; + mutable std::mutex mutex_; + std::condition_variable room_; + std::deque items_; + bool closed_ = false; +}; + +} // namespace oma diff --git a/tests/base/main.cpp b/tests/base/main.cpp index 48b248d..2f2abc2 100644 --- a/tests/base/main.cpp +++ b/tests/base/main.cpp @@ -5,6 +5,7 @@ void run_time_tests(); void run_error_tests(); void run_log_tests(); void run_job_tests(); +void run_bounded_queue_tests(); int main(int argc, char* argv[]) { cest_init(argc, argv); @@ -13,5 +14,6 @@ int main(int argc, char* argv[]) { run_error_tests(); run_log_tests(); run_job_tests(); + run_bounded_queue_tests(); return cest_result(); } diff --git a/tests/base/test_bounded_queue.cpp b/tests/base/test_bounded_queue.cpp new file mode 100644 index 0000000..eee4c5e --- /dev/null +++ b/tests/base/test_bounded_queue.cpp @@ -0,0 +1,101 @@ +#include "oma/base/bounded_queue.hpp" + +#include +#include +#include + +#include "oma_test.hpp" + +using oma::BoundedQueue; + +namespace { + +// A producer pushing 0..n-1 through a small queue while the consumer polls; returns the first +// out-of-order value, or -1. +long long stream_in_order(int n) { + BoundedQueue q(3); + std::thread producer([&] { + for (int i = 0; i < n; ++i) { + q.push(i); + } + }); + long long mismatch = -1; + int expected = 0; + while (expected < n) { + if (auto v = q.try_pop()) { + if (*v != expected && mismatch < 0) { + mismatch = expected; + } + ++expected; + } else { + std::this_thread::yield(); + } + } + producer.join(); + return mismatch; +} + +// Whether a producer blocked on a full queue stays blocked until there is room. +bool push_waits_for_room() { + BoundedQueue q(1); + q.push(1); + std::atomic pushed{false}; + std::thread producer([&] { + q.push(2); + pushed.store(true); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const bool blocked = !pushed.load(); + (void)q.try_pop(); + producer.join(); + return blocked && pushed.load() && q.size() == 1; +} + +// Whether close() releases a blocked producer and makes its push fail. +bool close_releases_producer() { + BoundedQueue q(1); + q.push(1); + std::atomic result{-1}; + std::thread producer([&] { result.store(q.push(2) ? 1 : 0); }); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + q.close(); + producer.join(); + return result.load() == 0 && q.size() == 1; +} + +} // namespace + +void run_bounded_queue_tests() { + describe("BoundedQueue", { + it("keeps order between a producer and a consumer thread", { + expect(stream_in_order(5000)).toBe(-1LL); + }); + + it("blocks the producer while full", { expect(push_waits_for_room()).toBeTruthy(); }); + + it("releases a blocked producer on close", { expect(close_releases_producer()).toBeTruthy(); }); + + it("pops only what the predicate accepts", { + BoundedQueue q(4); + q.push(5); + q.push(6); + expect(q.try_pop_if([](int v) { return v > 5; }).has_value()).toBeFalsy(); + expect(q.try_pop_if([](int v) { return v == 5; }).value_or(0)).toBe(5); + expect(q.size()).toBe(1U); + }); + + it("drops everything on clear and keeps accepting", { + BoundedQueue q(2); + q.push(1); + q.push(2); + q.clear(); + expect(q.size()).toBe(0U); + expect(q.push(3)).toBeTruthy(); + }); + + it("treats a zero capacity as one", { + BoundedQueue q(0); + expect(q.capacity()).toBe(1U); + }); + }); +} From 341ccbbb4b65e4020b6093d7780627a1eaa117b1 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:44 -0300 Subject: [PATCH 002/464] feat(gpu): allow zero-flag queues, instance extensions and a recursive queue lock --- libs/gpu/include/oma/gpu/device.hpp | 18 +++++++++--- libs/gpu/src/device.cpp | 27 +++++++++++++++--- tests/gpu/test_device.cpp | 43 +++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/libs/gpu/include/oma/gpu/device.hpp b/libs/gpu/include/oma/gpu/device.hpp index 18edd91..89aa2bd 100644 --- a/libs/gpu/include/oma/gpu/device.hpp +++ b/libs/gpu/include/oma/gpu/device.hpp @@ -40,7 +40,14 @@ struct DeviceInfo { struct DeviceOptions { // Pick the first device whose name contains this text (case-sensitive). Without it, the first // hardware device is used, falling back to a CPU implementation (e.g. lavapipe in CI). - std::optional name_contains; + std::optional name_contains = std::nullopt; + // Qt 6.11 retrieves queues with vkGetDeviceQueue, which requires zero creation flags. + // Disabling this requires every consumer to participate in external synchronization. + bool internally_synchronized_queues = true; + // Instance requirements of the presentation bridge or validation tools. Missing requested + // extensions/layers fail creation; libs/gpu does not depend on a window system or Qt. + std::vector instance_extensions = {}; + std::vector instance_layers = {}; }; // Lists the devices the loader exposes, without creating any logical device. @@ -61,7 +68,8 @@ class Device { [[nodiscard]] uint32_t graphics_family() const noexcept; // Whether queues were created internally synchronized (VK_KHR_internally_synchronized_queues): - // FFmpeg, the compositor and Qt can then submit to the same queue without an application lock. + // Compatible consumers can then submit without an application lock. Qt 6.11 cannot retrieve + // these flagged queues (S4); this does not imply compatibility with its device import API. [[nodiscard]] bool internally_synchronized_queues() const noexcept; [[nodiscard]] VkDeviceQueueCreateFlags queue_create_flags() const noexcept; @@ -80,8 +88,10 @@ class Device { [[nodiscard]] VkQueue queue(uint32_t family, uint32_t index) const; // Vulkan requires external synchronization of a VkQueue unless it was created internally - // synchronized. Every submitter (FFmpeg, the compositor, Qt) brackets vkQueueSubmit* and - // vkQueuePresent with these; they are no-ops on internally synchronized queues. + // synchronized. OmaMovie submitters bracket vkQueueSubmit* with these; they are no-ops on + // internally synchronized queues. Qt does not call these hooks: its bridge must serialize + // ALL queue access with other consumers (including submission, waits and presentation). + // The lock is recursive: a thread that already holds a queue may submit to it again. void lock_queue(uint32_t family, uint32_t index) const; void unlock_queue(uint32_t family, uint32_t index) const; diff --git a/libs/gpu/src/device.cpp b/libs/gpu/src/device.cpp index dbbdb7b..d9bffc1 100644 --- a/libs/gpu/src/device.cpp +++ b/libs/gpu/src/device.cpp @@ -32,6 +32,7 @@ Error vk_error(vk::Result r, std::string what) { // Extensions OmaMovie enables when the device has them. Interop and synchronization come first; // video extensions are only present on drivers that expose Vulkan Video (Docs/spikes/S1-*.md). constexpr std::array kFeaturelessExtensions = { + VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, @@ -79,7 +80,9 @@ struct Device::Impl { std::vector extensions; bool internally_synchronized = false; // One mutex per queue, indexed [family][index]; only used without internal synchronization. - mutable std::vector> queue_mutexes; + // Recursive: the Qt bridge holds the graphics queue for a whole frame (ADR-0005), and work + // recorded on the render thread inside that frame (the viewer's compositor) submits again. + mutable std::vector> queue_mutexes; // Enabled feature chain; FFmpeg keeps pointers into it, so Impl never moves (unique_ptr). vk::PhysicalDeviceFeatures2 features2{}; @@ -126,7 +129,22 @@ Result> Device::create(const DeviceOptions& options) { .pEngineName = "OmaMovie", .engineVersion = 1, .apiVersion = VK_API_VERSION_1_4}; - auto instance = impl->context.createInstance({.pApplicationInfo = &app}); + std::vector instance_extensions; + std::vector instance_layers; + instance_extensions.reserve(options.instance_extensions.size()); + instance_layers.reserve(options.instance_layers.size()); + for (const auto& extension : options.instance_extensions) { + instance_extensions.push_back(extension.c_str()); + } + for (const auto& layer : options.instance_layers) { + instance_layers.push_back(layer.c_str()); + } + auto instance = impl->context.createInstance( + {.pApplicationInfo = &app, + .enabledLayerCount = static_cast(instance_layers.size()), + .ppEnabledLayerNames = instance_layers.data(), + .enabledExtensionCount = static_cast(instance_extensions.size()), + .ppEnabledExtensionNames = instance_extensions.data()}); if (!instance) { return std::unexpected(vk_error(instance.error(), "cannot create a Vulkan instance")); } @@ -210,13 +228,14 @@ Result> Device::create(const DeviceOptions& options) { } vkGetPhysicalDeviceFeatures2(static_cast(*impl->physical), &static_cast(impl->features2)); - impl->internally_synchronized = impl->isq.internallySynchronizedQueues == vk::True; + impl->internally_synchronized = options.internally_synchronized_queues && + impl->isq.internallySynchronizedQueues == vk::True; if (impl->v12.timelineSemaphore != vk::True || impl->v13.synchronization2 != vk::True) { return make_error(ErrorCode::Unsupported, Category::Gpu, "timeline semaphores and synchronization2 are required", impl->info.name); } - // ---- queues: every queue of every family, internally synchronized when supported + // ---- queues: every queue of every family; flag them when supported and requested using QueueChain = vk::StructureChain; std::vector qprops; diff --git a/tests/gpu/test_device.cpp b/tests/gpu/test_device.cpp index bf224e4..a4dcb72 100644 --- a/tests/gpu/test_device.cpp +++ b/tests/gpu/test_device.cpp @@ -1,8 +1,11 @@ #include "oma/gpu/device.hpp" +#include +#include #include #include #include +#include #include "oma_test.hpp" @@ -37,6 +40,28 @@ oma::gpu::Device* gpu_test_device() { namespace { +// The Qt bridge holds the graphics queue for a frame and the render thread submits inside it; +// any other thread must still wait. +bool queue_lock_is_recursive_and_exclusive(const Device& d) { + const uint32_t family = d.graphics_family(); + d.lock_queue(family, 0); + d.lock_queue(family, 0); // re-entry on the same thread + std::atomic other_got_it{false}; + std::thread other([&] { + d.lock_queue(family, 0); + other_got_it.store(true); + d.unlock_queue(family, 0); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const bool excluded_while_held = !other_got_it.load(); + d.unlock_queue(family, 0); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + const bool still_excluded = !other_got_it.load(); // one level is still held + d.unlock_queue(family, 0); + other.join(); + return excluded_while_held && still_excluded && other_got_it.load(); +} + bool family_has(const Device& d, uint32_t family, VkQueueFlags flag) { for (const auto& f : d.queue_families()) { if (f.index == family) { @@ -116,6 +141,24 @@ void run_device_tests() { } expect(true).toBeTruthy(); }); + + it("can create zero-flag queues for consumers using vkGetDeviceQueue", { + auto compatible = Device::create({.internally_synchronized_queues = false}); + expect(compatible.has_value()).toBeTruthy(); + if (!compatible) { + return; + } + const auto& c = **compatible; + expect(c.internally_synchronized_queues()).toBeFalsy(); + expect(c.queue_create_flags()).toEqual(VkDeviceQueueCreateFlags{0}); + VkQueue retrieved = VK_NULL_HANDLE; + vkGetDeviceQueue(c.device(), c.graphics_family(), 0, &retrieved); + expect(retrieved != VK_NULL_HANDLE).toBeTruthy(); + expect(retrieved == c.queue(c.graphics_family(), 0)).toBeTruthy(); + c.lock_queue(c.graphics_family(), 0); + c.unlock_queue(c.graphics_family(), 0); + expect(queue_lock_is_recursive_and_exclusive(c)).toBeTruthy(); + }); }); describe("gpu::enumerate_devices", { From bf701448d7c4ef52b265ff42de55c3d1ccd1dd56 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:44 -0300 Subject: [PATCH 003/464] feat(media): convert software frames to rgba for thumbnails --- libs/media/include/oma/media/video_frame.hpp | 5 ++++ libs/media/module.mk | 2 +- libs/media/src/video_frame.cpp | 30 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/libs/media/include/oma/media/video_frame.hpp b/libs/media/include/oma/media/video_frame.hpp index ba634c1..cec0c57 100644 --- a/libs/media/include/oma/media/video_frame.hpp +++ b/libs/media/include/oma/media/video_frame.hpp @@ -111,6 +111,11 @@ class VideoFrame { [[nodiscard]] std::span plane(int index) const noexcept; [[nodiscard]] int stride(int index) const noexcept; + // Software preview bridge. Copies a decoded frame into a caller-owned RGBA8 buffer. + // The destination has at least `stride * height()` bytes; GPU frames are unsupported. + [[nodiscard]] Result copy_rgba(std::span destination, + int destination_stride) const; + // GPU frames only: locks the frame and returns its images (Unsupported on software frames). [[nodiscard]] Result acquire_gpu(); diff --git a/libs/media/module.mk b/libs/media/module.mk index b823610..daaa322 100644 --- a/libs/media/module.mk +++ b/libs/media/module.mk @@ -1,3 +1,3 @@ # libs/media — probe and decode on top of FFmpeg (libav* confined to src/). Depends on gpu, base. -MEDIA_PKGS := libavformat libavcodec libavutil libswresample +MEDIA_PKGS := libavformat libavcodec libavutil libswresample libswscale $(eval $(call oma_library,media,$(wildcard libs/media/src/*.cpp),gpu base,$(shell pkg-config --cflags $(MEDIA_PKGS) 2>/dev/null),$(shell pkg-config --libs $(MEDIA_PKGS) 2>/dev/null))) diff --git a/libs/media/src/video_frame.cpp b/libs/media/src/video_frame.cpp index d9fce3e..59530a6 100644 --- a/libs/media/src/video_frame.cpp +++ b/libs/media/src/video_frame.cpp @@ -7,9 +7,11 @@ extern "C" { #include #include #include +#include } #include +#include #include #include @@ -169,6 +171,34 @@ int VideoFrame::stride(int index) const noexcept { return impl_->frame->linesize[index]; } +Result VideoFrame::copy_rgba(std::span destination, + int destination_stride) const { + const AVFrame& frame = *impl_->frame; + if (on_gpu() || frame.width <= 0 || frame.height <= 0 || frame.width > INT_MAX / 4 || + destination_stride < frame.width * 4 || + destination.size() < + static_cast(destination_stride) * static_cast(frame.height)) { + return make_error(ErrorCode::InvalidArgument, Category::Decode, + "invalid software RGBA preview destination"); + } + auto* scale = sws_getContext( + frame.width, frame.height, static_cast(frame.format), frame.width, + frame.height, AV_PIX_FMT_RGBA, SWS_BILINEAR, nullptr, nullptr, nullptr); + if (scale == nullptr) { + return make_error(ErrorCode::Unsupported, Category::Decode, + "cannot convert this video pixel format to RGBA"); + } + const std::array data{destination.data(), nullptr, nullptr, nullptr}; + const std::array linesize{destination_stride, 0, 0, 0}; + const int rows = + sws_scale(scale, frame.data, frame.linesize, 0, frame.height, data.data(), linesize.data()); + sws_freeContext(scale); + if (rows != frame.height) { + return make_error(ErrorCode::Internal, Category::Decode, "RGBA preview conversion failed"); + } + return {}; +} + Result VideoFrame::acquire_gpu() { if (!on_gpu()) { return make_error(ErrorCode::Unsupported, Category::Decode, From 8c73c03dfafea674af3deef69db7aee011e4e7d5 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:44 -0300 Subject: [PATCH 004/464] feat(media): add a media importer and an export format checker --- libs/media/include/oma/media/format.hpp | 40 ++++++++++++++++++++ libs/media/src/format.cpp | 45 +++++++++++++++++++++++ tests/media/main.cpp | 2 + tests/media/test_format.cpp | 49 +++++++++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 libs/media/include/oma/media/format.hpp create mode 100644 libs/media/src/format.cpp create mode 100644 tests/media/test_format.cpp diff --git a/libs/media/include/oma/media/format.hpp b/libs/media/include/oma/media/format.hpp new file mode 100644 index 0000000..b38e219 --- /dev/null +++ b/libs/media/include/oma/media/format.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "oma/media/probe.hpp" + +#include +#include +#include + +namespace oma::media { + +// Names are FFmpeg muxer and encoder names (for example mp4, libx264, aac). This describes +// an intended output; it does not imply that a timeline has been rendered or a file written. +struct ExportFormat { + std::string container; + std::string video_encoder; + std::optional audio_encoder; +}; + +// Input formats and output capability checks vary independently. Neither interface exposes +// FFmpeg types to the editor; both are internal, in-process extension points. +class MediaImporter { +public: + virtual ~MediaImporter() = default; + [[nodiscard]] virtual Result + inspect_input(const std::filesystem::path& path) const = 0; +}; + +class ExportFormatChecker { +public: + virtual ~ExportFormatChecker() = default; + [[nodiscard]] virtual Result check_output(const ExportFormat& format) const = 0; +}; + +class FfmpegFormatBackend final : public MediaImporter, public ExportFormatChecker { +public: + [[nodiscard]] Result inspect_input(const std::filesystem::path& path) const override; + [[nodiscard]] Result check_output(const ExportFormat& format) const override; +}; + +} // namespace oma::media diff --git a/libs/media/src/format.cpp b/libs/media/src/format.cpp new file mode 100644 index 0000000..56928d2 --- /dev/null +++ b/libs/media/src/format.cpp @@ -0,0 +1,45 @@ +#include "oma/media/format.hpp" + +#include "ffmpeg.hpp" + +namespace oma::media { + +Result FfmpegFormatBackend::inspect_input(const std::filesystem::path& path) const { + return probe(path); +} + +Result FfmpegFormatBackend::check_output(const ExportFormat& format) const { + ff::ensure_initialized(); + if (format.container.empty() || format.video_encoder.empty() || + (format.audio_encoder && format.audio_encoder->empty())) { + return make_error(ErrorCode::InvalidArgument, Category::Encode, + "missing output format or encoder"); + } + const AVOutputFormat* muxer = av_guess_format(format.container.c_str(), nullptr, nullptr); + if (muxer == nullptr) { + return make_error(ErrorCode::Unsupported, Category::Encode, + "output container is unavailable", format.container); + } + const auto check_encoder = [&](const std::string& name, AVMediaType kind) -> Result { + const AVCodec* encoder = avcodec_find_encoder_by_name(name.c_str()); + if (encoder == nullptr || encoder->type != kind) { + return make_error(ErrorCode::Unsupported, Category::Encode, + "output encoder is unavailable", name); + } + if (avformat_query_codec(muxer, encoder->id, FF_COMPLIANCE_NORMAL) != 1) { + return make_error(ErrorCode::Unsupported, Category::Encode, + "encoder is not supported by the output container", + format.container + ":" + name); + } + return {}; + }; + if (auto checked = check_encoder(format.video_encoder, AVMEDIA_TYPE_VIDEO); !checked) { + return checked; + } + if (format.audio_encoder) { + return check_encoder(*format.audio_encoder, AVMEDIA_TYPE_AUDIO); + } + return {}; +} + +} // namespace oma::media diff --git a/tests/media/main.cpp b/tests/media/main.cpp index 6e7612b..cb028ff 100644 --- a/tests/media/main.cpp +++ b/tests/media/main.cpp @@ -10,6 +10,7 @@ #include "oma_test.hpp" void run_probe_tests(); +void run_format_tests(); void run_video_decoder_tests(); void run_gpu_decode_tests(); void run_audio_decoder_tests(); @@ -62,6 +63,7 @@ int main(int argc, char* argv[]) { oma::set_log_level(oma::Category::Media, oma::LogLevel::Off); cest_init(argc, argv); run_probe_tests(); + run_format_tests(); run_video_decoder_tests(); run_gpu_decode_tests(); run_audio_decoder_tests(); diff --git a/tests/media/test_format.cpp b/tests/media/test_format.cpp new file mode 100644 index 0000000..40160b5 --- /dev/null +++ b/tests/media/test_format.cpp @@ -0,0 +1,49 @@ +#include "oma/media/format.hpp" + +#include "media_test.hpp" + +#include + +#include "oma_test.hpp" + +void run_format_tests() { + describe("media format boundary", { + oma::media::FfmpegFormatBackend ffmpeg; + const oma::media::MediaImporter& importer = ffmpeg; + const oma::media::ExportFormatChecker& output = ffmpeg; + + it("uses the existing probe DTO for imported media", { + const auto result = importer.inspect_input(fixture("h264_30fps_aac.mp4")); + expect(result.has_value()).toBeTruthy(); + if (result) { + expect(result->best_video.has_value()).toBeTruthy(); + expect(result->best_audio.has_value()).toBeTruthy(); + } + }); + + it("accepts a muxer and installed video encoder that fit together", { + oma::media::ExportFormat format; + format.container = "mp4"; + format.video_encoder = "mpeg4"; + expect(output.check_output(format).has_value()).toBeTruthy(); + }); + + it("rejects missing and incompatible output formats", { + oma::media::ExportFormat missing; + missing.container = "not-a-muxer"; + missing.video_encoder = "mpeg4"; + auto wrong_kind = missing; + wrong_kind.container = "mp4"; + wrong_kind.video_encoder = "aac"; + auto incompatible = wrong_kind; + incompatible.container = "webm"; + incompatible.video_encoder = "mpeg4"; + expect(code_of(output.check_output(missing).error())) + .toEqual(static_cast(oma::ErrorCode::Unsupported)); + expect(code_of(output.check_output(wrong_kind).error())) + .toEqual(static_cast(oma::ErrorCode::Unsupported)); + expect(code_of(output.check_output(incompatible).error())) + .toEqual(static_cast(oma::ErrorCode::Unsupported)); + }); + }); +} From eddd8e2d16da68f3ab3820448b73b85414b92831 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:44 -0300 Subject: [PATCH 005/464] feat(compositor): encode the linear output for an sdr display --- .../include/oma/compositor/compositor.hpp | 11 ++ libs/compositor/shaders/display_srgb.comp | 24 +++ libs/compositor/src/vulkan_compositor.cpp | 184 ++++++++++++++++++ tests/compositor/test_compositors.cpp | 51 +++++ 4 files changed, 270 insertions(+) create mode 100644 libs/compositor/shaders/display_srgb.comp diff --git a/libs/compositor/include/oma/compositor/compositor.hpp b/libs/compositor/include/oma/compositor/compositor.hpp index a374254..6567ff6 100644 --- a/libs/compositor/include/oma/compositor/compositor.hpp +++ b/libs/compositor/include/oma/compositor/compositor.hpp @@ -71,6 +71,17 @@ class VulkanCompositor { // Copies the output back to memory (tests, thumbnails). Slow by design. [[nodiscard]] Result read_output(); + // Preview display transform: encodes the last render with the sRGB transfer function into + // an RGBA8 image (VK_IMAGE_LAYOUT_GENERAL, sampleable, same size as the output) for an SDR + // display. Returns when the GPU has finished; the image stays valid until the next call. + // Later work on this queue (for example Qt sampling it) is ordered after the encode, and the + // next encode waits for earlier work on the queue before overwriting it. The full SDR/HDR + // display policy is ADR-0006's. + [[nodiscard]] Result encode_display(); + + // RGBA8 pixels of the last encode_display() (tests). Slow by design. + [[nodiscard]] Result> read_display(); + private: struct Impl; explicit VulkanCompositor(std::unique_ptr impl); diff --git a/libs/compositor/shaders/display_srgb.comp b/libs/compositor/shaders/display_srgb.comp new file mode 100644 index 0000000..3a1788b --- /dev/null +++ b/libs/compositor/shaders/display_srgb.comp @@ -0,0 +1,24 @@ +#version 450 + +// Preview display transform (M3): the compositor's linear BT.709 RGBA16F image encoded with +// the sRGB transfer function into RGBA8 for an SDR display. Primaries are passed through +// (BT.709 = sRGB); the full display policy, including HDR, belongs to ADR-0006. +// Alpha is copied unchanged: the preview background is opaque. + +layout(local_size_x = 16, local_size_y = 16) in; +layout(rgba16f, set = 0, binding = 0) uniform readonly image2D linear_image; +layout(rgba8, set = 0, binding = 1) uniform writeonly image2D display_image; + +vec3 encode_srgb(vec3 linear_rgb) { + vec3 x = clamp(linear_rgb, vec3(0.0), vec3(1.0)); + vec3 low = 12.92 * x; + vec3 high = 1.055 * pow(x, vec3(1.0 / 2.4)) - 0.055; + return mix(high, low, lessThanEqual(x, vec3(0.0031308))); +} + +void main() { + ivec2 pixel = ivec2(gl_GlobalInvocationID.xy); + if (any(greaterThanEqual(pixel, imageSize(display_image)))) return; + vec4 linear_pixel = imageLoad(linear_image, pixel); + imageStore(display_image, pixel, vec4(encode_srgb(linear_pixel.rgb), linear_pixel.a)); +} diff --git a/libs/compositor/src/vulkan_compositor.cpp b/libs/compositor/src/vulkan_compositor.cpp index 9d664c4..41e77a7 100644 --- a/libs/compositor/src/vulkan_compositor.cpp +++ b/libs/compositor/src/vulkan_compositor.cpp @@ -24,6 +24,11 @@ namespace { constexpr auto kCompositeSpv = std::to_array({ #include "composite.comp.inc" }); +// SPIR-V of shaders/display_srgb.comp. +constexpr auto kDisplaySpv = std::to_array({ +#include "display_srgb.comp.inc" +}); +constexpr VkFormat kDisplayFormat = VK_FORMAT_R8G8B8A8_UNORM; constexpr std::uint32_t kGroupSize = 16; constexpr VkFormat kOutputFormat = VK_FORMAT_R16G16B16A16_SFLOAT; @@ -146,6 +151,13 @@ struct VulkanCompositor::Impl { PFN_vkCmdPushDescriptorSetKHR push_descriptors = nullptr; VkDeviceSize params_stride = 0; + // Display transform pass (encode_display). + VkDescriptorSetLayout display_set_layout = VK_NULL_HANDLE; + VkPipelineLayout display_pipeline_layout = VK_NULL_HANDLE; + VkPipeline display_pipeline = VK_NULL_HANDLE; + gpu::Image display; + gpu::Buffer display_readback; + gpu::Image output; gpu::Buffer params; gpu::Buffer readback; @@ -162,6 +174,9 @@ struct VulkanCompositor::Impl { return; } destroy_transient_views(); + vkDestroyPipeline(device->device(), display_pipeline, nullptr); + vkDestroyPipelineLayout(device->device(), display_pipeline_layout, nullptr); + vkDestroyDescriptorSetLayout(device->device(), display_set_layout, nullptr); vkDestroySampler(device->device(), sampler, nullptr); vkDestroyPipeline(device->device(), pipeline, nullptr); vkDestroyPipelineCache(device->device(), cache, nullptr); @@ -179,6 +194,7 @@ struct VulkanCompositor::Impl { [[nodiscard]] Result make_view(VkImage image, VkFormat format, VkImageAspectFlags aspect); [[nodiscard]] Result ensure_output(std::uint32_t width, std::uint32_t height); + [[nodiscard]] Result create_display_pipeline(); [[nodiscard]] Result ensure_params(std::size_t layers); [[nodiscard]] Result stage_upload(std::size_t index, const media::VideoFrame& frame); [[nodiscard]] Result bind_gpu_frame(media::VideoFrame& frame); @@ -226,6 +242,73 @@ Result VulkanCompositor::Impl::ensure_output(std::uint32_t width, std::uin return {}; } +// Two storage images (linear in, display out), bound with push descriptors like the compositor. +Result VulkanCompositor::Impl::create_display_pipeline() { + std::array bindings{}; + for (std::uint32_t i = 0; i < bindings.size(); ++i) { + bindings[i] = {.binding = i, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr}; + } + const VkDescriptorSetLayoutCreateInfo set_info{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = static_cast(bindings.size()), + .pBindings = bindings.data()}; + if (const VkResult r = + vkCreateDescriptorSetLayout(device->device(), &set_info, nullptr, &display_set_layout); + r != VK_SUCCESS) { + return std::unexpected(vk_failure(r, "cannot create the display descriptor layout")); + } + const VkPipelineLayoutCreateInfo layout_info{.sType = + VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .setLayoutCount = 1, + .pSetLayouts = &display_set_layout, + .pushConstantRangeCount = 0, + .pPushConstantRanges = nullptr}; + if (const VkResult r = vkCreatePipelineLayout(device->device(), &layout_info, nullptr, + &display_pipeline_layout); + r != VK_SUCCESS) { + return std::unexpected(vk_failure(r, "cannot create the display pipeline layout")); + } + const VkShaderModuleCreateInfo module_info{.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .codeSize = sizeof(kDisplaySpv), + .pCode = kDisplaySpv.data()}; + VkShaderModule module = VK_NULL_HANDLE; + if (const VkResult r = vkCreateShaderModule(device->device(), &module_info, nullptr, &module); + r != VK_SUCCESS) { + return std::unexpected(vk_failure(r, "cannot load the display shader")); + } + const VkComputePipelineCreateInfo pipe_info{ + .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, + .module = module, + .pName = "main", + .pSpecializationInfo = nullptr}, + .layout = display_pipeline_layout, + .basePipelineHandle = VK_NULL_HANDLE, + .basePipelineIndex = -1}; + const VkResult made = vkCreateComputePipelines(device->device(), cache, 1, &pipe_info, nullptr, + &display_pipeline); + vkDestroyShaderModule(device->device(), module, nullptr); + if (made != VK_SUCCESS) { + return std::unexpected(vk_failure(made, "cannot create the display pipeline")); + } + return {}; +} + Result VulkanCompositor::Impl::ensure_params(std::size_t layers) { const VkDeviceSize needed = params_stride * std::max(layers, 1); if (params.size() >= needed) { @@ -458,6 +541,9 @@ Result> VulkanCompositor::create(const gpu::De r != VK_SUCCESS) { return std::unexpected(vk_failure(r, "cannot create the sampler")); } + if (auto r = impl->create_display_pipeline(); !r) { + return std::unexpected(r.error()); + } return std::unique_ptr(new VulkanCompositor(std::move(impl))); } @@ -716,4 +802,102 @@ Result VulkanCompositor::read_output() { return out; } +Result VulkanCompositor::encode_display() { + Impl& d = *impl_; + if (d.output.handle() == VK_NULL_HANDLE) { + return make_error(ErrorCode::InvalidArgument, Category::Compositor, "nothing rendered yet"); + } + const std::uint32_t w = d.output.desc().width; + const std::uint32_t h = d.output.desc().height; + if (d.display.handle() == VK_NULL_HANDLE || d.display.desc().width != w || + d.display.desc().height != h) { + auto img = gpu::Image::create( + *d.device, {.width = w, + .height = h, + .format = kDisplayFormat, + .usage = static_cast(VK_IMAGE_USAGE_STORAGE_BIT) | + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT}); + if (!img) { + return std::unexpected(img.error()); + } + d.display = std::move(*img); + } + const auto ran = d.runner->run([&](VkCommandBuffer cmd) { + // Waits for every earlier command on this queue (a reader of the previous encode) before + // the contents are discarded. + gpu::transition(cmd, d.display.handle(), VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_GENERAL); + vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, d.display_pipeline); + const VkDescriptorImageInfo source{.sampler = VK_NULL_HANDLE, + .imageView = d.output.view(), + .imageLayout = VK_IMAGE_LAYOUT_GENERAL}; + const VkDescriptorImageInfo target{.sampler = VK_NULL_HANDLE, + .imageView = d.display.view(), + .imageLayout = VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + for (std::uint32_t b = 0; b < writes.size(); ++b) { + writes[b] = {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = VK_NULL_HANDLE, + .dstBinding = b, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .pImageInfo = b == 0 ? &source : &target, + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr}; + } + d.push_descriptors(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, d.display_pipeline_layout, 0, + static_cast(writes.size()), writes.data()); + vkCmdDispatch(cmd, (w + kGroupSize - 1) / kGroupSize, (h + kGroupSize - 1) / kGroupSize, 1); + // Visible to any later reader on the queue (Qt's fragment shader, a copy). + barrier(cmd, + memory_barrier(VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_WRITE_BIT, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT)); + }); + if (!ran) { + return std::unexpected(ran.error()); + } + return &d.display; +} + +Result> VulkanCompositor::read_display() { + Impl& d = *impl_; + if (d.display.handle() == VK_NULL_HANDLE) { + return make_error(ErrorCode::InvalidArgument, Category::Compositor, "nothing encoded yet"); + } + const std::uint32_t w = d.display.desc().width; + const std::uint32_t h = d.display.desc().height; + const VkDeviceSize bytes = static_cast(w) * h * 4; + if (d.display_readback.size() < bytes) { + auto buf = gpu::Buffer::create(*d.device, bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT, + gpu::MemoryUse::Readback); + if (!buf) { + return std::unexpected(buf.error()); + } + d.display_readback = std::move(*buf); + } + const auto ran = d.runner->run([&](VkCommandBuffer cmd) { + const VkBufferImageCopy copy{.bufferOffset = 0, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource = {.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1}, + .imageOffset = {.x = 0, .y = 0, .z = 0}, + .imageExtent = {.width = w, .height = h, .depth = 1}}; + vkCmdCopyImageToBuffer(cmd, d.display.handle(), VK_IMAGE_LAYOUT_GENERAL, + d.display_readback.handle(), 1, ©); + barrier(cmd, memory_barrier(VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, + VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT)); + }); + if (!ran) { + return std::unexpected(ran.error()); + } + d.display_readback.invalidate(); + const auto* data = static_cast(d.display_readback.mapped()); + return std::vector(data, data + bytes); +} + } // namespace oma::compositor diff --git a/tests/compositor/test_compositors.cpp b/tests/compositor/test_compositors.cpp index 7950aa5..9932045 100644 --- a/tests/compositor/test_compositors.cpp +++ b/tests/compositor/test_compositors.cpp @@ -4,6 +4,7 @@ #include "compositor_test.hpp" +#include #include #include #include @@ -179,6 +180,55 @@ void gpu_matches_cpu_with_uploads() { expect(diff.over_tolerance <= cpu->pixels.size() / 4 / 500).toBeTruthy(); } +// The sRGB transfer function (IEC 61966-2-1), as the display pass applies it. +int srgb_level(float linear) { + const double x = std::clamp(static_cast(linear), 0.0, 1.0); + const double v = x <= 0.0031308 ? 12.92 * x : (1.055 * std::pow(x, 1.0 / 2.4)) - 0.055; + return static_cast(std::lround(v * 255.0)); +} + +// The display pass encodes the linear output with the sRGB curve, within one 8-bit level, and +// keeps alpha. +void display_encodes_srgb() { + const oma::gpu::Device* device = compositor_test_device(); + if (device == nullptr) { + return; + } + auto a = decode_first("h264_30fps_aac.mp4"); + auto b = decode_first("hevc_10bit.mp4"); + if (!a.frame || !b.frame) { + std::printf(" (skipped: fixtures missing)\n"); + return; + } + const std::array inputs{a.input(), b.input()}; + auto vk = VulkanCompositor::create(*device); + expect(vk.has_value()).toBeTruthy(); + if (!vk) { + return; + } + const RenderGraph g = busy_graph(320, 180); + expect((*vk)->render(g, inputs).has_value()).toBeTruthy(); + const auto encoded = (*vk)->encode_display(); + expect(encoded.has_value()).toBeTruthy(); + const auto linear = (*vk)->read_output(); + const auto display = (*vk)->read_display(); + expect(linear && display).toBeTruthy(); + if (!linear || !display) { + return; + } + expect(display->size() == linear->pixels.size()).toBeTruthy(); + int worst = 0; + for (std::size_t i = 0; i < display->size(); ++i) { + const int expected = + i % 4 == 3 + ? static_cast(std::lround(std::clamp(linear->pixels[i], 0.0F, 1.0F) * 255.0F)) + : srgb_level(linear->pixels[i]); + worst = std::max(worst, std::abs(static_cast((*display)[i]) - expected)); + } + std::printf(" worst difference %d of 255\n", worst); + expect(worst <= 1).toBeTruthy(); +} + void gpu_frames_match_cpu_reference() { const oma::gpu::Device* device = compositor_test_device(); if (device == nullptr) { @@ -265,6 +315,7 @@ void run_vulkan_compositor_tests() { it("matches the CPU reference on uploaded frames", { gpu_matches_cpu_with_uploads(); }); it("matches the CPU reference on zero-copy GPU frames", { gpu_frames_match_cpu_reference(); }); + it("encodes the output for an SDR display", { display_encodes_srgb(); }); it("measures three 1080p layers", { measures_1080p_three_layers(); }); }); } From 340c3f3ad0d492b7ec1b2e1cf86c5bf44b30d020 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:45 -0300 Subject: [PATCH 006/464] test(fixtures): add a single-frame y4m still --- tests/fixtures/generate.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/fixtures/generate.sh b/tests/fixtures/generate.sh index 23ca098..3071fed 100755 --- a/tests/fixtures/generate.sh +++ b/tests/fixtures/generate.sh @@ -96,6 +96,10 @@ note tone_44100.wav # Still image. ff -f lavfi -i "testsrc2=size=640x360:rate=1:duration=1" -frames:v 1 "$out/still.png" note still.png +# The compositor currently accepts YUV video frames, not RGB PNG frames. A single-frame Y4M +# preserves the still-image scenario for its GPU path until RGB image import lands. +ff -i "$out/still.png" -frames:v 1 -pix_fmt yuv420p -f yuv4mpegpipe "$out/still.y4m" +note still.y4m echo "Fixtures in $out:" printf ' %s\n' "${made[@]}" From 78545d1eb4ab3f2d1c1ba2861456180d184f25b3 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:57 -0300 Subject: [PATCH 007/464] feat(audio): add the sample ring, playback clock, mixer and pipewire output --- Makefile | 2 + PKGBUILD | 3 +- libs/audio/include/oma/audio/clock.hpp | 43 ++++ libs/audio/include/oma/audio/mix.hpp | 30 +++ libs/audio/include/oma/audio/output.hpp | 68 ++++++ libs/audio/include/oma/audio/ring.hpp | 46 ++++ libs/audio/module.mk | 4 + libs/audio/src/clock.cpp | 24 ++ libs/audio/src/mix.cpp | 51 +++++ libs/audio/src/null_output.cpp | 114 ++++++++++ libs/audio/src/pipewire_output.cpp | 277 ++++++++++++++++++++++++ libs/audio/src/ring.cpp | 55 +++++ tests/audio/main.cpp | 15 ++ tests/audio/module.mk | 3 + tests/audio/test_clock.cpp | 64 ++++++ tests/audio/test_mix.cpp | 90 ++++++++ tests/audio/test_output.cpp | 156 +++++++++++++ tests/audio/test_ring.cpp | 112 ++++++++++ tests/support/lsan.supp | 6 + 19 files changed, 1162 insertions(+), 1 deletion(-) create mode 100644 libs/audio/include/oma/audio/clock.hpp create mode 100644 libs/audio/include/oma/audio/mix.hpp create mode 100644 libs/audio/include/oma/audio/output.hpp create mode 100644 libs/audio/include/oma/audio/ring.hpp create mode 100644 libs/audio/module.mk create mode 100644 libs/audio/src/clock.cpp create mode 100644 libs/audio/src/mix.cpp create mode 100644 libs/audio/src/null_output.cpp create mode 100644 libs/audio/src/pipewire_output.cpp create mode 100644 libs/audio/src/ring.cpp create mode 100644 tests/audio/main.cpp create mode 100644 tests/audio/module.mk create mode 100644 tests/audio/test_clock.cpp create mode 100644 tests/audio/test_mix.cpp create mode 100644 tests/audio/test_output.cpp create mode 100644 tests/audio/test_ring.cpp diff --git a/Makefile b/Makefile index 473305e..1e3a9bc 100644 --- a/Makefile +++ b/Makefile @@ -129,10 +129,12 @@ include libs/base/module.mk include libs/gpu/module.mk include libs/media/module.mk include libs/compositor/module.mk +include libs/audio/module.mk include tests/base/module.mk include tests/gpu/module.mk include tests/media/module.mk include tests/compositor/module.mk +include tests/audio/module.mk # -------------------------------------------------------------------------- rules diff --git a/PKGBUILD b/PKGBUILD index 0c95b61..e6d69a5 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -24,7 +24,8 @@ depends=( 'gcc-libs' # libstdc++ (libs/base) 'glibc' 'vulkan-icd-loader' # libvulkan (libs/gpu) - 'ffmpeg' # libavformat/libavcodec/libavutil/libswresample (libs/media) + 'ffmpeg' # libavformat/libavcodec/libavutil/libswresample/libswscale (libs/media) + 'libpipewire' # PipeWire client (libs/audio) ) makedepends=( 'git' diff --git a/libs/audio/include/oma/audio/clock.hpp b/libs/audio/include/oma/audio/clock.hpp new file mode 100644 index 0000000..4f807a8 --- /dev/null +++ b/libs/audio/include/oma/audio/clock.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/time.hpp" + +#include + +// Playback master clock (CLAUDE.md §12): the position that is audible now, derived from the +// frames the device actually consumed, never from wall time. Video follows it by dropping or +// repeating frames. +// +// There is no separate monotonic fallback: the timeline mix always produces samples (silence +// where no clip plays), and machines without an audio device consume them through the null +// output, which is paced by a steady clock. + +namespace oma::audio { + +class PlaybackClock { +public: + explicit PlaybackClock(SampleRate rate) noexcept : rate_(rate) {} + + [[nodiscard]] SampleRate rate() const noexcept { return rate_; } + + // Ties a media position to the device's consumed count, after a seek (output flushed) or + // before the first write. The position is rounded to the nearest sample. + [[nodiscard]] Result anchor(const RationalTime& position, std::int64_t frames_consumed); + + // The audible position: the anchor plus the frames consumed since then, minus the frames + // still buffered in the device. It never goes before the anchor (latency larger than what + // was consumed means nothing from this anchor is audible yet). Expressed in samples. + [[nodiscard]] RationalTime position(std::int64_t frames_consumed, + std::int64_t latency_frames) const noexcept; + + // The anchor in samples (the first sample written after it). + [[nodiscard]] std::int64_t anchor_sample() const noexcept { return anchor_sample_; } + +private: + SampleRate rate_; + std::int64_t anchor_sample_ = 0; + std::int64_t anchor_consumed_ = 0; +}; + +} // namespace oma::audio diff --git a/libs/audio/include/oma/audio/mix.hpp b/libs/audio/include/oma/audio/mix.hpp new file mode 100644 index 0000000..0360d3b --- /dev/null +++ b/libs/audio/include/oma/audio/mix.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +// Mixing primitives for timeline audio (CLAUDE.md §12): plain arithmetic on float samples, no +// allocation, usable from any thread. Positions and lengths are in samples, never seconds. + +namespace oma::audio { + +// A clip's gain over its length: a constant gain with linear fades at both ends. +struct ClipGain { + float gain = 1.0F; + std::int64_t length = 0; // clip length in samples + std::int64_t fade_in = 0; // samples + std::int64_t fade_out = 0; // samples + + // The gain at `sample` (0 = the clip's first sample). Zero outside the clip. + [[nodiscard]] float at(std::int64_t sample) const noexcept; +}; + +// Adds `frames` samples of planar input into interleaved output, scaled by the clip's gain at +// each sample. `input` holds `input_channels` planes of at least `frames` samples each, plane c +// at input[c * stride ...]. Mono input feeds every output channel; extra input channels beyond +// the output's are dropped (no downmix matrix yet). +void mix_planar(std::span output, int output_channels, std::span input, + int input_channels, std::int64_t stride, std::int64_t frames, const ClipGain& gain, + std::int64_t first_clip_sample) noexcept; + +} // namespace oma::audio diff --git a/libs/audio/include/oma/audio/output.hpp b/libs/audio/include/oma/audio/output.hpp new file mode 100644 index 0000000..d45831d --- /dev/null +++ b/libs/audio/include/oma/audio/output.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include "oma/audio/ring.hpp" +#include "oma/base/error.hpp" +#include "oma/base/time.hpp" + +#include +#include +#include +#include + +// Audio output (CLAUDE.md §12). The device pulls interleaved float frames from a SampleRing on its +// real-time thread; the count of frames it consumed is the playback master clock. + +namespace oma::audio { + +struct OutputFormat { + SampleRate rate = SampleRate::make(48000).value(); + int channels = 2; +}; + +class AudioOutput { +public: + AudioOutput() = default; + AudioOutput(const AudioOutput&) = delete; + AudioOutput& operator=(const AudioOutput&) = delete; + AudioOutput(AudioOutput&&) = delete; + AudioOutput& operator=(AudioOutput&&) = delete; + virtual ~AudioOutput() = default; + + [[nodiscard]] virtual const OutputFormat& format() const noexcept = 0; + // The producer side belongs to one playback thread. + [[nodiscard]] virtual SampleRing& ring() noexcept = 0; + + [[nodiscard]] virtual Result start() = 0; + virtual void stop() noexcept = 0; + + // Real frames taken from the ring since creation (silence for underruns is not counted, so + // the clock stalls instead of running ahead of the audio). Thread-safe, monotonic. + [[nodiscard]] virtual std::int64_t frames_consumed() const noexcept = 0; + // Frames taken from the ring but not yet audible (device buffering), for A/V sync. + [[nodiscard]] virtual std::int64_t latency_frames() const noexcept = 0; + // Callbacks that found the ring short of data. + [[nodiscard]] virtual std::int64_t underruns() const noexcept = 0; + + // Drops queued frames (seek). Returns once the consumer has dropped them; the producer must + // not write meanwhile. + virtual void flush() noexcept = 0; + + // Silences the output without changing its timing: the device keeps consuming at its rate, + // so the clock stays real (measurements, a preview mute). + virtual void set_muted(bool muted) noexcept = 0; + // False once the output can no longer play (the stream failed or lost its device). Playback + // then restarts on a new output from the audible position. + [[nodiscard]] virtual bool healthy() const noexcept = 0; +}; + +// Consumes frames in real time without making sound, paced by a steady clock: tests, CI and +// machines without PipeWire. +[[nodiscard]] Result> +make_null_output(const OutputFormat& format, + std::chrono::milliseconds period = std::chrono::milliseconds(10)); + +// A PipeWire playback stream. Unsupported when no PipeWire daemon answers. +[[nodiscard]] Result> +make_pipewire_output(const OutputFormat& format, std::string_view node_name = "OmaMovie"); + +} // namespace oma::audio diff --git a/libs/audio/include/oma/audio/ring.hpp b/libs/audio/include/oma/audio/ring.hpp new file mode 100644 index 0000000..6551d2d --- /dev/null +++ b/libs/audio/include/oma/audio/ring.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include +#include + +// Lock-free single-producer/single-consumer ring of interleaved float frames (CLAUDE.md §12). +// After construction nothing allocates or blocks, so the real-time audio thread can read from it. + +namespace oma::audio { + +class SampleRing { +public: + // Capacity is rounded up to a power of two frames. + SampleRing(std::size_t capacity_frames, int channels); + + SampleRing(const SampleRing&) = delete; + SampleRing& operator=(const SampleRing&) = delete; + SampleRing(SampleRing&&) = delete; + SampleRing& operator=(SampleRing&&) = delete; + ~SampleRing() = default; + + [[nodiscard]] int channels() const noexcept { return channels_; } + [[nodiscard]] std::size_t capacity() const noexcept { return mask_ + 1; } + + // Producer side. Writes whole frames; returns how many fit. + std::size_t write(std::span interleaved) noexcept; + [[nodiscard]] std::size_t writable() const noexcept; + + // Consumer side. Reads whole frames; returns how many were available. + std::size_t read(std::span interleaved) noexcept; + [[nodiscard]] std::size_t readable() const noexcept; + // Drops everything readable now (consumer side, e.g. when a flush was requested). + void discard_readable() noexcept; + +private: + int channels_; + std::size_t mask_; + std::vector data_; + alignas(64) std::atomic head_{0}; // frames written (producer) + alignas(64) std::atomic tail_{0}; // frames read (consumer) +}; + +} // namespace oma::audio diff --git a/libs/audio/module.mk b/libs/audio/module.mk new file mode 100644 index 0000000..55d1ebc --- /dev/null +++ b/libs/audio/module.mk @@ -0,0 +1,4 @@ +# libs/audio — sample ring, audio clock and outputs (PipeWire, null). Depends on base only. +# PipeWire/SPA headers are C with heavy macros: include them as system headers. +AUDIO_PKGS := libpipewire-0.3 +$(eval $(call oma_library,audio,$(wildcard libs/audio/src/*.cpp),base,$(patsubst -I%,-isystem %,$(shell pkg-config --cflags $(AUDIO_PKGS) 2>/dev/null)),$(shell pkg-config --libs $(AUDIO_PKGS) 2>/dev/null))) diff --git a/libs/audio/src/clock.cpp b/libs/audio/src/clock.cpp new file mode 100644 index 0000000..42bf8ce --- /dev/null +++ b/libs/audio/src/clock.cpp @@ -0,0 +1,24 @@ +#include "oma/audio/clock.hpp" + +#include + +namespace oma::audio { + +Result PlaybackClock::anchor(const RationalTime& position, std::int64_t frames_consumed) { + auto sample = rate_.time_to_sample(position, Rounding::Nearest); + if (!sample) { + return std::unexpected(sample.error()); + } + anchor_sample_ = *sample; + anchor_consumed_ = frames_consumed; + return {}; +} + +RationalTime PlaybackClock::position(std::int64_t frames_consumed, + std::int64_t latency_frames) const noexcept { + const std::int64_t audible = + std::max(frames_consumed - anchor_consumed_ - latency_frames, 0); + return rate_.sample_to_time(anchor_sample_ + audible); +} + +} // namespace oma::audio diff --git a/libs/audio/src/mix.cpp b/libs/audio/src/mix.cpp new file mode 100644 index 0000000..eadc7bd --- /dev/null +++ b/libs/audio/src/mix.cpp @@ -0,0 +1,51 @@ +#include "oma/audio/mix.hpp" + +#include +#include +#include + +namespace oma::audio { + +float ClipGain::at(std::int64_t sample) const noexcept { + if (sample < 0 || sample >= length) { + return 0.0F; + } + float g = gain; + if (fade_in > 0 && sample < fade_in) { + g *= static_cast(sample) / static_cast(fade_in); + } + const std::int64_t remaining = length - sample; // 1 at the last sample + if (fade_out > 0 && remaining <= fade_out) { + g *= static_cast(remaining - 1) / static_cast(fade_out); + } + return g; +} + +void mix_planar(std::span output, int output_channels, std::span input, + int input_channels, std::int64_t stride, std::int64_t frames, const ClipGain& gain, + std::int64_t first_clip_sample) noexcept { + if (output_channels <= 0 || input_channels <= 0 || frames <= 0) { + return; + } + const auto out_ch = static_cast(output_channels); + const auto count = + std::min(static_cast(frames), output.size() / out_ch); + for (std::size_t i = 0; i < count; ++i) { + const float g = gain.at(first_clip_sample + static_cast(i)); + if (g == 0.0F) { + continue; + } + for (std::size_t c = 0; c < out_ch; ++c) { + const std::size_t plane = input_channels == 1 ? 0 : c; + if (std::cmp_greater_equal(plane, input_channels)) { + continue; + } + const std::size_t at = (plane * static_cast(stride)) + i; + if (at < input.size()) { + output[(i * out_ch) + c] += input[at] * g; + } + } + } +} + +} // namespace oma::audio diff --git a/libs/audio/src/null_output.cpp b/libs/audio/src/null_output.cpp new file mode 100644 index 0000000..5372942 --- /dev/null +++ b/libs/audio/src/null_output.cpp @@ -0,0 +1,114 @@ +#include "oma/audio/output.hpp" + +#include +#include +#include + +namespace oma::audio { + +namespace { + +constexpr std::size_t kRingSeconds = 2; + +class NullOutput final : public AudioOutput { +public: + NullOutput(const OutputFormat& format, std::chrono::milliseconds period) + : format_(format), period_(period), + ring_(static_cast(format.rate.hz()) * kRingSeconds, format.channels) {} + + NullOutput(const NullOutput&) = delete; + NullOutput& operator=(const NullOutput&) = delete; + NullOutput(NullOutput&&) = delete; + NullOutput& operator=(NullOutput&&) = delete; + ~NullOutput() override { stop(); } + + [[nodiscard]] const OutputFormat& format() const noexcept override { return format_; } + [[nodiscard]] SampleRing& ring() noexcept override { return ring_; } + + [[nodiscard]] Result start() override { + if (thread_.joinable()) { + return {}; + } + thread_ = std::jthread([this](const std::stop_token& stop) { run(stop); }); + return {}; + } + + void stop() noexcept override { + if (thread_.joinable()) { + thread_.request_stop(); + thread_.join(); + } + } + + [[nodiscard]] std::int64_t frames_consumed() const noexcept override { + return consumed_.load(std::memory_order_acquire); + } + [[nodiscard]] std::int64_t latency_frames() const noexcept override { return 0; } + [[nodiscard]] std::int64_t underruns() const noexcept override { + return underruns_.load(std::memory_order_relaxed); + } + + void set_muted(bool /*muted*/) noexcept override {} + [[nodiscard]] bool healthy() const noexcept override { return true; } + + void flush() noexcept override { + if (!thread_.joinable()) { + ring_.discard_readable(); + return; + } + flush_requested_.store(true, std::memory_order_release); + while (flush_requested_.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + +private: + // The "device": every period, consume the frames that elapsed in real time. + void run(const std::stop_token& stop) { + const auto rate = static_cast(format_.rate.hz()); + std::vector scratch; + const auto start = std::chrono::steady_clock::now(); + std::int64_t due_total = 0; + while (!stop.stop_requested()) { + if (flush_requested_.load(std::memory_order_acquire)) { + ring_.discard_readable(); + flush_requested_.store(false, std::memory_order_release); + } + const auto elapsed = std::chrono::steady_clock::now() - start; + const std::int64_t due = + std::chrono::duration_cast(elapsed).count() * rate / + 1'000'000; + const auto frames = static_cast(due - due_total); + due_total = due; + scratch.resize(frames * static_cast(format_.channels)); + const std::size_t got = ring_.read(scratch); + if (got < frames) { + underruns_.fetch_add(1, std::memory_order_relaxed); + } + consumed_.fetch_add(static_cast(got), std::memory_order_release); + std::this_thread::sleep_for(period_); + } + // A flush requested while stopping must not wait forever. + flush_requested_.store(false, std::memory_order_release); + } + + OutputFormat format_; + std::chrono::milliseconds period_; + SampleRing ring_; + std::atomic consumed_{0}; + std::atomic underruns_{0}; + std::atomic flush_requested_{false}; + std::jthread thread_; +}; + +} // namespace + +Result> make_null_output(const OutputFormat& format, + std::chrono::milliseconds period) { + if (format.channels <= 0 || period.count() <= 0) { + return make_error(ErrorCode::InvalidArgument, Category::Audio, "invalid output format"); + } + return std::unique_ptr(std::make_unique(format, period)); +} + +} // namespace oma::audio diff --git a/libs/audio/src/pipewire_output.cpp b/libs/audio/src/pipewire_output.cpp new file mode 100644 index 0000000..9cf8c29 --- /dev/null +++ b/libs/audio/src/pipewire_output.cpp @@ -0,0 +1,277 @@ +#include "oma/audio/output.hpp" + +#include "oma/base/log.hpp" + +// PipeWire and SPA are C headers full of macros; they are included as system headers (see +// module.mk) and only used in this file. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace oma::audio { + +namespace { + +constexpr std::size_t kRingSeconds = 2; +constexpr int kConnectTimeoutSeconds = 2; + +class PipeWireOutput final : public AudioOutput { +public: + explicit PipeWireOutput(const OutputFormat& format) + : format_(format), + ring_(static_cast(format.rate.hz()) * kRingSeconds, format.channels) {} + + PipeWireOutput(const PipeWireOutput&) = delete; + PipeWireOutput& operator=(const PipeWireOutput&) = delete; + PipeWireOutput(PipeWireOutput&&) = delete; + PipeWireOutput& operator=(PipeWireOutput&&) = delete; + + ~PipeWireOutput() override { + if (loop_ != nullptr) { + pw_thread_loop_stop(loop_); + } + if (stream_ != nullptr) { + pw_stream_destroy(stream_); + } + if (loop_ != nullptr) { + pw_thread_loop_destroy(loop_); + } + } + + [[nodiscard]] Result connect(std::string_view node_name); + + [[nodiscard]] const OutputFormat& format() const noexcept override { return format_; } + [[nodiscard]] SampleRing& ring() noexcept override { return ring_; } + + [[nodiscard]] Result start() override { + pw_thread_loop_lock(loop_); + const int r = pw_stream_set_active(stream_, true); + pw_thread_loop_unlock(loop_); + if (r < 0) { + return make_error(ErrorCode::Internal, Category::Audio, + "cannot start the PipeWire stream", + std::error_code(-r, std::generic_category()).message()); + } + active_.store(true, std::memory_order_release); + return {}; + } + + void stop() noexcept override { + pw_thread_loop_lock(loop_); + pw_stream_set_active(stream_, false); + pw_thread_loop_unlock(loop_); + active_.store(false, std::memory_order_release); + } + + [[nodiscard]] std::int64_t frames_consumed() const noexcept override { + return consumed_.load(std::memory_order_acquire); + } + [[nodiscard]] std::int64_t latency_frames() const noexcept override { + return latency_.load(std::memory_order_relaxed); + } + [[nodiscard]] std::int64_t underruns() const noexcept override { + return underruns_.load(std::memory_order_relaxed); + } + + void set_muted(bool muted) noexcept override { + float value = muted ? 1.0F : 0.0F; // the C API takes a mutable array + pw_thread_loop_lock(loop_); + pw_stream_set_control(stream_, SPA_PROP_mute, 1, &value, 0); + pw_thread_loop_unlock(loop_); + } + + [[nodiscard]] bool healthy() const noexcept override { + const pw_stream_state s = state_.load(std::memory_order_acquire); + return s != PW_STREAM_STATE_ERROR && s != PW_STREAM_STATE_UNCONNECTED; + } + + void flush() noexcept override { + if (!active_.load(std::memory_order_acquire)) { + ring_.discard_readable(); + return; + } + flush_requested_.store(true, std::memory_order_release); + // The real-time thread handles it within one quantum; give up after a second in case + // the graph stopped pulling (suspended sink). + for (int i = 0; i < 1000 && flush_requested_.load(std::memory_order_acquire); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + if (flush_requested_.exchange(false, std::memory_order_acq_rel)) { + ring_.discard_readable(); + } + } + +private: + static void on_process(void* data) { static_cast(data)->process(); } + + static void on_state_changed(void* data, pw_stream_state /*old*/, pw_stream_state state, + const char* error) { + auto* self = static_cast(data); + self->state_.store(state, std::memory_order_release); + if (state == PW_STREAM_STATE_ERROR && error != nullptr) { + log_warn(Category::Audio, "PipeWire stream error: {}", error); + } + pw_thread_loop_signal(self->loop_, false); + } + + // Real-time thread: no allocation, no locks, no logging. + void process() noexcept { + pw_buffer* b = pw_stream_dequeue_buffer(stream_); + if (b == nullptr) { + return; + } + const spa_data& d = b->buffer->datas[0]; + if (d.data == nullptr) { + pw_stream_queue_buffer(stream_, b); + return; + } + if (flush_requested_.load(std::memory_order_acquire)) { + ring_.discard_readable(); + flush_requested_.store(false, std::memory_order_release); + } + const auto stride = static_cast(sizeof(float)) * + static_cast(format_.channels); + std::uint64_t frames = d.maxsize / stride; + if (b->requested != 0) { + frames = std::min(frames, b->requested); + } + const std::span out(static_cast(d.data), + frames * static_cast(format_.channels)); + const std::size_t got = ring_.read(out); + if (got < frames) { + std::fill(out.begin() + static_cast( + got * static_cast(format_.channels)), + out.end(), 0.0F); + underruns_.fetch_add(1, std::memory_order_relaxed); + } + consumed_.fetch_add(static_cast(got), std::memory_order_release); + + pw_time time{}; + if (pw_stream_get_time_n(stream_, &time, sizeof time) == 0 && time.rate.denom != 0) { + // delay is in 1/rate.denom ticks; convert to output frames. + const auto delay = static_cast(time.delay) * format_.rate.hz() * + static_cast(time.rate.num) / + static_cast(time.rate.denom); + latency_.store(delay + static_cast(time.queued), + std::memory_order_relaxed); + } + + d.chunk->offset = 0; + d.chunk->stride = static_cast(stride); + d.chunk->size = static_cast(frames) * stride; + b->size = frames; + pw_stream_queue_buffer(stream_, b); + } + + OutputFormat format_; + SampleRing ring_; + pw_thread_loop* loop_ = nullptr; + pw_stream* stream_ = nullptr; + pw_stream_events events_{}; + std::atomic state_{PW_STREAM_STATE_UNCONNECTED}; + std::atomic active_{false}; + std::atomic flush_requested_{false}; + std::atomic consumed_{0}; + std::atomic latency_{0}; + std::atomic underruns_{0}; +}; + +Result PipeWireOutput::connect(std::string_view node_name) { + static std::once_flag init; + std::call_once(init, [] { pw_init(nullptr, nullptr); }); + + loop_ = pw_thread_loop_new("oma-audio", nullptr); + if (loop_ == nullptr) { + return make_error(ErrorCode::Internal, Category::Audio, "cannot create the PipeWire loop"); + } + const std::string name(node_name); + pw_properties* props = pw_properties_new( + PW_KEY_MEDIA_TYPE, "Audio", PW_KEY_MEDIA_CATEGORY, "Playback", PW_KEY_MEDIA_ROLE, + "Production", PW_KEY_APP_NAME, "OmaMovie", PW_KEY_NODE_NAME, name.c_str(), nullptr); + events_.version = PW_VERSION_STREAM_EVENTS; + events_.state_changed = on_state_changed; + events_.process = on_process; + stream_ = + pw_stream_new_simple(pw_thread_loop_get_loop(loop_), name.c_str(), props, &events_, this); + if (stream_ == nullptr) { + return make_error(ErrorCode::Unsupported, Category::Audio, + "cannot create a PipeWire stream"); + } + + std::array pod_buffer{}; + spa_pod_builder builder{}; + spa_pod_builder_init(&builder, pod_buffer.data(), + static_cast(pod_buffer.size())); + spa_audio_info_raw info{}; + info.format = SPA_AUDIO_FORMAT_F32; + info.rate = static_cast(format_.rate.hz()); + info.channels = static_cast(format_.channels); + if (format_.channels == 2) { + info.position[0] = SPA_AUDIO_CHANNEL_FL; + info.position[1] = SPA_AUDIO_CHANNEL_FR; + } else if (format_.channels == 1) { + info.position[0] = SPA_AUDIO_CHANNEL_MONO; + } else { + info.flags = SPA_AUDIO_FLAG_UNPOSITIONED; + } + std::array params{ + spa_format_audio_raw_build(&builder, SPA_PARAM_EnumFormat, &info)}; + + pw_thread_loop_lock(loop_); + if (pw_thread_loop_start(loop_) < 0) { + pw_thread_loop_unlock(loop_); + return make_error(ErrorCode::Internal, Category::Audio, "cannot start the PipeWire loop"); + } + const auto flags = + static_cast(static_cast(PW_STREAM_FLAG_AUTOCONNECT) | + static_cast(PW_STREAM_FLAG_MAP_BUFFERS) | + static_cast(PW_STREAM_FLAG_RT_PROCESS) | + static_cast(PW_STREAM_FLAG_INACTIVE)); + const int r = + pw_stream_connect(stream_, PW_DIRECTION_OUTPUT, PW_ID_ANY, flags, params.data(), 1); + // Wait until the stream is ready (paused, inactive) or failed; state changes signal the loop. + for (int waited = 0; r >= 0 && waited < kConnectTimeoutSeconds; ++waited) { + const pw_stream_state s = state_.load(std::memory_order_acquire); + if (s == PW_STREAM_STATE_PAUSED || s == PW_STREAM_STATE_STREAMING || + s == PW_STREAM_STATE_ERROR) { + break; + } + pw_thread_loop_timed_wait(loop_, 1); // seconds; releases the lock while waiting + } + pw_thread_loop_unlock(loop_); + const pw_stream_state s = state_.load(std::memory_order_acquire); + if (r < 0 || (s != PW_STREAM_STATE_PAUSED && s != PW_STREAM_STATE_STREAMING)) { + return make_error(ErrorCode::Unsupported, Category::Audio, "no PipeWire output available", + r < 0 ? std::error_code(-r, std::generic_category()).message() + : pw_stream_state_as_string(s)); + } + log_info(Category::Audio, "PipeWire output: {} Hz, {} channels", format_.rate.hz(), + format_.channels); + return {}; +} + +} // namespace + +Result> make_pipewire_output(const OutputFormat& format, + std::string_view node_name) { + if (format.channels <= 0 || std::cmp_greater(format.channels, SPA_AUDIO_MAX_CHANNELS)) { + return make_error(ErrorCode::InvalidArgument, Category::Audio, "invalid output format"); + } + auto out = std::make_unique(format); + if (auto r = out->connect(node_name); !r) { + return std::unexpected(r.error()); + } + return std::unique_ptr(std::move(out)); +} + +} // namespace oma::audio diff --git a/libs/audio/src/ring.cpp b/libs/audio/src/ring.cpp new file mode 100644 index 0000000..965af8b --- /dev/null +++ b/libs/audio/src/ring.cpp @@ -0,0 +1,55 @@ +#include "oma/audio/ring.hpp" + +#include +#include + +namespace oma::audio { + +SampleRing::SampleRing(std::size_t capacity_frames, int channels) + : channels_(std::max(channels, 1)), + mask_(std::bit_ceil(std::max(capacity_frames, 2)) - 1), + data_((mask_ + 1) * static_cast(channels_)) {} + +std::size_t SampleRing::writable() const noexcept { + const std::uint64_t head = head_.load(std::memory_order_relaxed); + const std::uint64_t tail = tail_.load(std::memory_order_acquire); + return capacity() - static_cast(head - tail); +} + +std::size_t SampleRing::readable() const noexcept { + const std::uint64_t tail = tail_.load(std::memory_order_relaxed); + const std::uint64_t head = head_.load(std::memory_order_acquire); + return static_cast(head - tail); +} + +std::size_t SampleRing::write(std::span interleaved) noexcept { + const auto ch = static_cast(channels_); + const std::size_t frames = std::min(interleaved.size() / ch, writable()); + const std::uint64_t head = head_.load(std::memory_order_relaxed); + for (std::size_t i = 0; i < frames; ++i) { + const std::size_t slot = static_cast(head + i) & mask_; + std::copy_n(interleaved.begin() + static_cast(i * ch), ch, + data_.begin() + static_cast(slot * ch)); + } + head_.store(head + frames, std::memory_order_release); + return frames; +} + +std::size_t SampleRing::read(std::span interleaved) noexcept { + const auto ch = static_cast(channels_); + const std::size_t frames = std::min(interleaved.size() / ch, readable()); + const std::uint64_t tail = tail_.load(std::memory_order_relaxed); + for (std::size_t i = 0; i < frames; ++i) { + const std::size_t slot = static_cast(tail + i) & mask_; + std::copy_n(data_.begin() + static_cast(slot * ch), ch, + interleaved.begin() + static_cast(i * ch)); + } + tail_.store(tail + frames, std::memory_order_release); + return frames; +} + +void SampleRing::discard_readable() noexcept { + tail_.store(head_.load(std::memory_order_acquire), std::memory_order_release); +} + +} // namespace oma::audio diff --git a/tests/audio/main.cpp b/tests/audio/main.cpp new file mode 100644 index 0000000..f871f50 --- /dev/null +++ b/tests/audio/main.cpp @@ -0,0 +1,15 @@ +#include "oma_test.hpp" + +void run_ring_tests(); +void run_clock_tests(); +void run_output_tests(); +void run_mix_tests(); + +int main(int argc, char* argv[]) { + cest_init(argc, argv); + run_ring_tests(); + run_clock_tests(); + run_output_tests(); + run_mix_tests(); + return cest_result(); +} diff --git a/tests/audio/module.mk b/tests/audio/module.mk new file mode 100644 index 0000000..d61580d --- /dev/null +++ b/tests/audio/module.mk @@ -0,0 +1,3 @@ +# libs/audio tests: the sample ring, the master clock and the outputs (the PipeWire case skips +# when no daemon answers, as in CI). +$(eval $(call oma_test,audio_tests,$(wildcard tests/audio/*.cpp),audio base)) diff --git a/tests/audio/test_clock.cpp b/tests/audio/test_clock.cpp new file mode 100644 index 0000000..e86871c --- /dev/null +++ b/tests/audio/test_clock.cpp @@ -0,0 +1,64 @@ +#include "oma/audio/clock.hpp" + +#include "oma_test.hpp" + +using oma::Rational; +using oma::RationalTime; +using oma::SampleRate; +using oma::audio::PlaybackClock; + +namespace { + +SampleRate hz(int rate) { + return SampleRate::make(rate).value(); +} + +RationalTime at(std::int64_t value, std::int64_t num, std::int64_t den) { + return RationalTime::make(value, Rational::make(num, den).value()).value(); +} + +} // namespace + +void run_clock_tests() { + describe("PlaybackClock", { + it("starts at the anchor and advances with consumed frames", { + PlaybackClock clock(hz(48000)); + expect(clock.anchor(at(0, 1, 1), 0).has_value()).toBeTruthy(); + expect(clock.position(0, 0) == at(0, 1, 1)).toBeTruthy(); + expect(clock.position(48000, 0) == at(1, 1, 1)).toBeTruthy(); + }); + + it("subtracts the device latency", { + PlaybackClock clock(hz(48000)); + expect(clock.anchor(at(10, 1, 1), 1000).has_value()).toBeTruthy(); + // 1000 + 48000 consumed, 2400 still buffered: 10 s + 45600 samples. + expect(clock.position(49000, 2400) == at(10 * 48000 + 45600, 1, 48000)).toBeTruthy(); + }); + + it("never reports a position before the anchor", { + PlaybackClock clock(hz(44100)); + expect(clock.anchor(at(5, 1, 1), 500).has_value()).toBeTruthy(); + expect(clock.position(600, 1024) == at(5, 1, 1)).toBeTruthy(); + }); + + it("rounds a video anchor to the nearest sample", { + PlaybackClock clock(hz(44100)); + // Frame 1 at 29.97 fps is 1001/30000 s = 1471.47 samples at 44.1 kHz. + expect(clock.anchor(at(1, 1001, 30000), 0).has_value()).toBeTruthy(); + expect(clock.anchor_sample()).toBe(1471LL); + }); + + it("re-anchors after a seek relative to the new consumed count", { + PlaybackClock clock(hz(96000)); + expect(clock.anchor(at(0, 1, 1), 0).has_value()).toBeTruthy(); + expect(clock.anchor(at(3600, 1, 1), 192000).has_value()).toBeTruthy(); + expect(clock.position(288000, 0) == at(3601, 1, 1)).toBeTruthy(); + }); + + it("reports overflow for an anchor beyond the sample range", { + PlaybackClock clock(hz(96000)); + const auto huge = at(INT64_MAX / 2, 1, 1); + expect(clock.anchor(huge, 0).has_value()).toBeFalsy(); + }); + }); +} diff --git a/tests/audio/test_mix.cpp b/tests/audio/test_mix.cpp new file mode 100644 index 0000000..c80b623 --- /dev/null +++ b/tests/audio/test_mix.cpp @@ -0,0 +1,90 @@ +#include "oma/audio/mix.hpp" + +#include + +#include "oma_test.hpp" + +using oma::audio::ClipGain; +using oma::audio::mix_planar; + +namespace { + +ClipGain faded() { + return ClipGain{.gain = 0.5F, .length = 100, .fade_in = 10, .fade_out = 20}; +} + +// Two planes of 4 samples: left 1, 2, 3, 4 and right 10, 20, 30, 40. +std::vector stereo_planes() { + return {1, 2, 3, 4, 10, 20, 30, 40}; +} + +ClipGain unity(std::int64_t length) { + return ClipGain{.gain = 1.0F, .length = length, .fade_in = 0, .fade_out = 0}; +} + +std::vector mono_planes() { + return {3, 5}; +} + +} // namespace + +void run_mix_tests() { + describe("ClipGain", { + it("is constant between the fades", { + expect(faded().at(50)).toBe(0.5F); + expect(faded().at(10)).toBe(0.5F); + expect(faded().at(79)).toBe(0.5F); + }); + + it("ramps linearly in from silence", { + expect(faded().at(0)).toBe(0.0F); + expect(faded().at(5)).toBe(0.25F); + }); + + it("ramps out to silence at the last sample", { + expect(faded().at(99)).toBe(0.0F); + expect(faded().at(89)).toBe(0.25F); + }); + + it("is silent outside the clip", { + expect(faded().at(-1)).toBe(0.0F); + expect(faded().at(100)).toBe(0.0F); + }); + }); + + describe("mix_planar", { + it("interleaves planar stereo and adds to what is there", { + std::vector out(8, 1.0F); + const auto in = stereo_planes(); + mix_planar(out, 2, in, 2, 4, 4, unity(4), 0); + expect(out[0]).toBe(2.0F); + expect(out[1]).toBe(11.0F); + expect(out[6]).toBe(5.0F); + expect(out[7]).toBe(41.0F); + }); + + it("feeds mono to every output channel", { + std::vector out(4, 0.0F); + const auto in = mono_planes(); + mix_planar(out, 2, in, 1, 2, 2, unity(2), 0); + expect(out[0] == 3.0F && out[1] == 3.0F).toBeTruthy(); + expect(out[2] == 5.0F && out[3] == 5.0F).toBeTruthy(); + }); + + it("applies the gain at the clip position of each sample", { + std::vector out(8, 0.0F); + const auto in = stereo_planes(); + // Samples 98..101 of a 100-sample clip: only 98 is still inside (fading out). + mix_planar(out, 2, in, 2, 4, 4, faded(), 98); + expect(out[0]).toBe(0.025F); + expect(out[2] == 0.0F && out[4] == 0.0F && out[6] == 0.0F).toBeTruthy(); + }); + + it("never writes past the output", { + std::vector out(2, 0.0F); + const auto in = stereo_planes(); + mix_planar(out, 2, in, 2, 4, 4, unity(4), 0); + expect(out[0] == 1.0F && out[1] == 10.0F).toBeTruthy(); + }); + }); +} diff --git a/tests/audio/test_output.cpp b/tests/audio/test_output.cpp new file mode 100644 index 0000000..a01a051 --- /dev/null +++ b/tests/audio/test_output.cpp @@ -0,0 +1,156 @@ +#include "oma/audio/output.hpp" + +#include +#include +#include +#include +#include + +#include "oma_test.hpp" + +using oma::audio::AudioOutput; +using oma::audio::OutputFormat; + +namespace { + +using std::chrono::milliseconds; + +OutputFormat stereo48k() { + return OutputFormat{}; +} + +std::unique_ptr null_output() { + return oma::audio::make_null_output(stereo48k(), milliseconds(2)).value(); +} + +void write_silence(AudioOutput& out, std::size_t frames) { + const std::vector silence(frames * static_cast(out.format().channels), + 0.0F); + out.ring().write(silence); +} + +// Polls until `done` holds or the deadline passes (sanitizer builds run slowly). +template +bool wait_for(Pred done, milliseconds deadline = milliseconds(5000)) { + const auto until = std::chrono::steady_clock::now() + deadline; + while (!done()) { + if (std::chrono::steady_clock::now() > until) { + return false; + } + std::this_thread::sleep_for(milliseconds(1)); + } + return true; +} + +bool null_consumes_everything_written() { + auto out = null_output(); + write_silence(*out, 2400); // 50 ms + if (!out->start()) { + return false; + } + const bool drained = wait_for([&] { return out->frames_consumed() == 2400; }); + out->stop(); + return drained && out->ring().readable() == 0; +} + +bool null_counts_underruns_without_advancing() { + auto out = null_output(); + if (!out->start()) { + return false; + } + const bool underran = wait_for([&] { return out->underruns() > 0; }); + out->stop(); + return underran && out->frames_consumed() == 0; +} + +bool null_flush_drops_queued_frames() { + auto out = null_output(); + write_silence(*out, 48000); // one second, far more than the test runs + if (!out->start()) { + return false; + } + out->flush(); + const bool empty = out->ring().readable() == 0; + const std::int64_t after_flush = out->frames_consumed(); + std::this_thread::sleep_for(milliseconds(20)); + const bool stalled = out->frames_consumed() == after_flush; + out->stop(); + return empty && stalled; +} + +bool null_consumes_at_the_device_rate() { + auto out = null_output(); + write_silence(*out, 48000); + const auto start = std::chrono::steady_clock::now(); + if (!out->start()) { + return false; + } + std::this_thread::sleep_for(milliseconds(100)); + const std::int64_t consumed = out->frames_consumed(); + const auto elapsed = std::chrono::steady_clock::now() - start; + out->stop(); + const auto elapsed_frames = + std::chrono::duration_cast(elapsed).count() * 48 / 1000; + // Never ahead of real time; at least half of it even on a loaded sanitizer run. + return consumed <= elapsed_frames && consumed >= 4800 / 2; +} + +} // namespace + +void run_output_tests() { + describe("NullOutput", { + it("rejects an invalid format", { + OutputFormat bad; + bad.channels = 0; + expect(oma::audio::make_null_output(bad).has_value()).toBeFalsy(); + }); + + it("consumes everything written", + { expect(null_consumes_everything_written()).toBeTruthy(); }); + + it("counts underruns and stalls the clock without data", + { expect(null_counts_underruns_without_advancing()).toBeTruthy(); }); + + it("drops queued frames on flush", + { expect(null_flush_drops_queued_frames()).toBeTruthy(); }); + + it("consumes frames no faster than real time", + { expect(null_consumes_at_the_device_rate()).toBeTruthy(); }); + + it("flushes and stops safely when never started", { + auto out = null_output(); + write_silence(*out, 100); + out->flush(); + out->stop(); + expect(out->ring().readable()).toBe(0U); + }); + }); + + describe("PipeWireOutput", { + it("plays silence through the daemon when one answers", { + auto out = oma::audio::make_pipewire_output(stereo48k(), "OmaMovie tests"); + if (!out) { + std::printf(" (skipping PipeWire test: %s)\n", out.error().summary().c_str()); + return; + } + write_silence(**out, 4800); + expect((*out)->start().has_value()).toBeTruthy(); + // A suspended or busy graph may not pull at once; report instead of failing. + const bool advanced = + wait_for([&] { return (*out)->frames_consumed() > 0; }, milliseconds(2000)); + std::printf(" consumed %lld frames, latency %lld, underruns %lld\n", + static_cast((*out)->frames_consumed()), + static_cast((*out)->latency_frames()), + static_cast((*out)->underruns())); + if (!advanced) { + std::printf(" (the graph did not pull: no sink or suspended)\n"); + } + expect((*out)->healthy()).toBeTruthy(); + (*out)->set_muted(true); // the test plays silence anyway; this exercises the control + (*out)->flush(); + (*out)->stop(); + // Silence padded for underruns is never counted as consumed. + expect((*out)->frames_consumed() <= 4800).toBeTruthy(); + }); + }); +} diff --git a/tests/audio/test_ring.cpp b/tests/audio/test_ring.cpp new file mode 100644 index 0000000..f14d89c --- /dev/null +++ b/tests/audio/test_ring.cpp @@ -0,0 +1,112 @@ +#include "oma/audio/ring.hpp" + +#include +#include +#include +#include + +#include "oma_test.hpp" + +using oma::audio::SampleRing; + +namespace { + +std::vector ramp(std::size_t count, float first) { + std::vector v(count); + for (std::size_t i = 0; i < count; ++i) { + v[i] = first + static_cast(i); + } + return v; +} + +// Streams `total` stereo frames (values 0, 1, 2, ... per sample) from a producer thread through a +// small ring and checks the consumer receives them in order. Returns the first mismatch index +// or -1. +long long stream_through_ring(std::size_t total) { + SampleRing ring(64, 2); + std::thread producer([&] { + std::size_t sent = 0; + while (sent < total) { + const std::size_t chunk = std::min(37, total - sent); + std::vector data(chunk * 2); + for (std::size_t i = 0; i < data.size(); ++i) { + data[i] = static_cast((sent * 2) + i); + } + sent += ring.write(data); + std::this_thread::yield(); + } + }); + long long mismatch = -1; + std::size_t received = 0; + std::vector buffer(2 * 23); + while (received < total) { + const std::size_t got = ring.read(buffer); + for (std::size_t i = 0; i < got * 2 && mismatch < 0; ++i) { + if (buffer[i] != static_cast((received * 2) + i)) { + mismatch = static_cast((received * 2) + i); + } + } + received += got; + std::this_thread::yield(); + } + producer.join(); + return mismatch; +} + +} // namespace + +void run_ring_tests() { + describe("SampleRing", { + it("rounds the capacity up to a power of two frames", { + SampleRing ring(100, 2); + expect(ring.capacity()).toBe(128U); + expect(ring.writable()).toBe(128U); + expect(ring.readable()).toBe(0U); + }); + + it("writes and reads whole interleaved frames in order", { + SampleRing ring(8, 2); + const auto in = ramp(6, 0.0F); + expect(ring.write(in)).toBe(3U); + std::vector out(6); + expect(ring.read(out)).toBe(3U); + expect(out[0]).toBe(0.0F); + expect(out[5]).toBe(5.0F); + }); + + it("ignores a trailing partial frame", { + SampleRing ring(8, 2); + const auto in = ramp(5, 0.0F); + expect(ring.write(in)).toBe(2U); + expect(ring.readable()).toBe(2U); + }); + + it("accepts only what fits and keeps the rest for later", { + SampleRing ring(4, 1); + const auto in = ramp(6, 10.0F); + expect(ring.write(in)).toBe(4U); + expect(ring.writable()).toBe(0U); + std::vector out(2); + expect(ring.read(out)).toBe(2U); + expect(out[1]).toBe(11.0F); + const auto more = ramp(3, 20.0F); + expect(ring.write(more)).toBe(2U); + std::vector rest(8); + expect(ring.read(rest)).toBe(4U); + expect(rest[0]).toBe(12.0F); + expect(rest[3]).toBe(21.0F); + }); + + it("drops everything readable on discard", { + SampleRing ring(8, 2); + const auto in = ramp(8, 0.0F); + ring.write(in); + ring.discard_readable(); + expect(ring.readable()).toBe(0U); + expect(ring.writable()).toBe(8U); + }); + + it("keeps order between a producer and a consumer thread", + { expect(stream_through_ring(20000)).toBe(-1LL); }); + }); +} diff --git a/tests/support/lsan.supp b/tests/support/lsan.supp index e60c475..10ae508 100644 --- a/tests/support/lsan.supp +++ b/tests/support/lsan.supp @@ -21,3 +21,9 @@ leak:oma::media::ff::wrap_vulkan_device leak:iHD_drv_video.so leak:libigdgmm.so leak:libva.so + +# PipeWire 1.x: client modules loaded while creating a stream's context (pw_context_load_module, +# e.g. module-rt talking to RTKit over D-Bus) keep allocations after pw_stream_destroy; the module +# is unloaded by then, so the frames are unsymbolized. Only reachable where a PipeWire daemon +# answers (not in CI). +leak:pw_context_load_module From e6fc2b5fe5038677a72a64b66024b1fcab1d4131 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:37:57 -0300 Subject: [PATCH 008/464] feat(timeline): add the timeline model, edit commands with undo/redo and evaluation --- Makefile | 2 + libs/timeline/include/oma/timeline/edit.hpp | 93 +++ libs/timeline/include/oma/timeline/editor.hpp | 81 +++ .../include/oma/timeline/evaluate.hpp | 48 ++ libs/timeline/include/oma/timeline/ids.hpp | 39 ++ libs/timeline/include/oma/timeline/model.hpp | 211 +++++++ libs/timeline/module.mk | 3 + libs/timeline/src/commands.cpp | 475 +++++++++++++++ libs/timeline/src/commands.hpp | 39 ++ libs/timeline/src/edit.cpp | 570 ++++++++++++++++++ libs/timeline/src/editor.cpp | 96 +++ libs/timeline/src/evaluate.cpp | 64 ++ libs/timeline/src/model.cpp | 278 +++++++++ libs/timeline/src/mutation.hpp | 51 ++ libs/timeline/src/time_math.cpp | 109 ++++ tests/timeline/fixture.hpp | 108 ++++ tests/timeline/main.cpp | 13 + tests/timeline/module.mk | 3 + tests/timeline/test_edit.cpp | 448 ++++++++++++++ tests/timeline/test_evaluate.cpp | 128 ++++ tests/timeline/test_model.cpp | 160 +++++ 21 files changed, 3019 insertions(+) create mode 100644 libs/timeline/include/oma/timeline/edit.hpp create mode 100644 libs/timeline/include/oma/timeline/editor.hpp create mode 100644 libs/timeline/include/oma/timeline/evaluate.hpp create mode 100644 libs/timeline/include/oma/timeline/ids.hpp create mode 100644 libs/timeline/include/oma/timeline/model.hpp create mode 100644 libs/timeline/module.mk create mode 100644 libs/timeline/src/commands.cpp create mode 100644 libs/timeline/src/commands.hpp create mode 100644 libs/timeline/src/edit.cpp create mode 100644 libs/timeline/src/editor.cpp create mode 100644 libs/timeline/src/evaluate.cpp create mode 100644 libs/timeline/src/model.cpp create mode 100644 libs/timeline/src/mutation.hpp create mode 100644 libs/timeline/src/time_math.cpp create mode 100644 tests/timeline/fixture.hpp create mode 100644 tests/timeline/main.cpp create mode 100644 tests/timeline/module.mk create mode 100644 tests/timeline/test_edit.cpp create mode 100644 tests/timeline/test_evaluate.cpp create mode 100644 tests/timeline/test_model.cpp diff --git a/Makefile b/Makefile index 1e3a9bc..b01aa5f 100644 --- a/Makefile +++ b/Makefile @@ -130,11 +130,13 @@ include libs/gpu/module.mk include libs/media/module.mk include libs/compositor/module.mk include libs/audio/module.mk +include libs/timeline/module.mk include tests/base/module.mk include tests/gpu/module.mk include tests/media/module.mk include tests/compositor/module.mk include tests/audio/module.mk +include tests/timeline/module.mk # -------------------------------------------------------------------------- rules diff --git a/libs/timeline/include/oma/timeline/edit.hpp b/libs/timeline/include/oma/timeline/edit.hpp new file mode 100644 index 0000000..2038831 --- /dev/null +++ b/libs/timeline/include/oma/timeline/edit.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include "oma/base/rational.hpp" +#include "oma/base/time.hpp" +#include "oma/timeline/editor.hpp" +#include "oma/timeline/ids.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include + +// Editing operations, each a Command for Editor::execute. Times are timeline times and must +// lie on the sequence grid (Timeline::to_ticks); source times are exact media times. +// +// Ripple operations shift the clips that follow on the same track only. Magnetism (closing +// gaps on a primary storyline) is a UI policy built from these generic commands +// (implementation plan M5), not a mode of the model. + +namespace oma::timeline::edit { + +// What a new clip shows: `duration` of timeline time starting at media time `source_in`. +struct ClipSource { + MediaId media; + RationalTime source_in; + RationalTime duration; // timeline time + TimeMap time_map; + VideoProperties video; + AudioProperties audio; +}; + +// Inserts the track at `index` (default: on top of the tracks of its kind order, i.e. last). +[[nodiscard]] std::unique_ptr add_track(TrackId id, TrackKind kind, std::string name, + std::optional index = std::nullopt); +// Removes a track and its clips. +[[nodiscard]] std::unique_ptr remove_track(TrackId id); +[[nodiscard]] std::unique_ptr set_track_flags(TrackId id, bool muted, bool hidden); + +// Places a clip at `at`, replacing whatever the range covers on the track (clips are trimmed, +// split or removed). +[[nodiscard]] std::unique_ptr overwrite(TrackId track, ClipId id, RationalTime at, + ClipSource source); +// Places a clip at `at` and pushes everything from `at` onward later by its duration; a clip +// spanning `at` is split. +[[nodiscard]] std::unique_ptr insert(TrackId track, ClipId id, RationalTime at, + ClipSource source); +// Places a clip right after the last clip of the track. +[[nodiscard]] std::unique_ptr append(TrackId track, ClipId id, ClipSource source); + +// Removes a clip and leaves a gap (lift). +[[nodiscard]] std::unique_ptr remove_clip(ClipId id); +// Removes a clip and closes the gap. +[[nodiscard]] std::unique_ptr ripple_delete(ClipId id); +// Removes the timeline range [from, to) from a track, trimming or splitting clips, and closes +// the gap. +[[nodiscard]] std::unique_ptr ripple_delete_range(TrackId track, RationalTime from, + RationalTime to); +// Splits a clip at a time strictly inside it; the right part gets a new ID. +[[nodiscard]] std::unique_ptr split(ClipId id, RationalTime at); + +// Moves the clip's first instant to `new_start`, keeping its last instant (source in moves +// with it). With ripple, the clip stays in place and the clips after it move instead. +[[nodiscard]] std::unique_ptr trim_start(ClipId id, RationalTime new_start, bool ripple); +// Moves the clip's end to `new_end`. With ripple, the clips after it follow. +[[nodiscard]] std::unique_ptr trim_end(ClipId id, RationalTime new_end, bool ripple); +// Moves the edit point between a clip and the clip that starts exactly at its end. +[[nodiscard]] std::unique_ptr roll(ClipId left, RationalTime edit_point); +// Shows different media through the same timeline range: source in moves by `delta` of +// timeline time (scaled by the clip's speed). +[[nodiscard]] std::unique_ptr slip(ClipId id, RationalTime delta); +// Moves a clip by `delta` between its neighbours: an adjacent previous clip's end and an +// adjacent next clip's start follow; gaps absorb the move otherwise. +[[nodiscard]] std::unique_ptr slide(ClipId id, RationalTime delta); +// Moves a clip to another time and/or track (same kind); the destination must be free. +[[nodiscard]] std::unique_ptr move_clip(ClipId id, TrackId track, RationalTime start); + +// Changes a constant speed keeping the same source range; the duration becomes +// source length / speed, rounded down to the sequence grid. With ripple, later clips follow. +[[nodiscard]] std::unique_ptr set_speed(ClipId id, Rational speed, bool ripple); +[[nodiscard]] std::unique_ptr set_video(ClipId id, VideoProperties video); +[[nodiscard]] std::unique_ptr set_audio(ClipId id, AudioProperties audio); + +[[nodiscard]] std::unique_ptr add_marker(MarkerId id, RationalTime at, std::string name); +[[nodiscard]] std::unique_ptr remove_marker(MarkerId id); + +// Several commands as one history entry, applied in order; each sees the state left by the +// previous ones. All or nothing. +[[nodiscard]] std::unique_ptr transaction(std::string name, + std::vector> commands); + +} // namespace oma::timeline::edit diff --git a/libs/timeline/include/oma/timeline/editor.hpp b/libs/timeline/include/oma/timeline/editor.hpp new file mode 100644 index 0000000..72e4d00 --- /dev/null +++ b/libs/timeline/include/oma/timeline/editor.hpp @@ -0,0 +1,81 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/timeline/ids.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include + +// Commands and history (CLAUDE.md §10): every edit is a Command applied through +// Editor::execute, the single entry point. Undo/redo restores exact prior states; compound +// operations are one history entry. +// +// Threading: an Editor and its Timeline belong to one thread (the UI thread, ADR-0003). +// Readers on other threads get copies, never references. + +namespace oma::timeline { + +class Command { +public: + Command() = default; + Command(const Command&) = delete; + Command& operator=(const Command&) = delete; + Command(Command&&) = delete; + Command& operator=(Command&&) = delete; + virtual ~Command() = default; + + // Shown in the history ("Undo Split"). + [[nodiscard]] virtual std::string_view name() const noexcept = 0; + // Applies the edit. On failure the timeline is left exactly as it was. + [[nodiscard]] virtual Result apply(Timeline& timeline) = 0; + // Undoes a successful apply. Called only in reverse order of application, so the state it + // finds is the one its apply left behind. + virtual void revert(Timeline& timeline) noexcept = 0; +}; + +class Editor { +public: + explicit Editor(Timeline timeline) noexcept : timeline_(std::move(timeline)) {} + + [[nodiscard]] const Timeline& timeline() const noexcept { return timeline_; } + + // Registers media for clips to reference. Importing is not a timeline edit: it stays out + // of the history (undoing an edit never unregisters media a library still shows), and + // registered media is never removed while the timeline lives. + [[nodiscard]] Result add_media(MediaInfo media); + + // Applies a command, checks every timeline invariant and records it for undo; clears the + // redo stack. A failing command or a broken invariant leaves the timeline unchanged. + [[nodiscard]] Result execute(std::unique_ptr command); + + [[nodiscard]] bool can_undo() const noexcept { return !undo_.empty(); } + [[nodiscard]] bool can_redo() const noexcept { return !redo_.empty(); } + [[nodiscard]] std::string_view undo_name() const noexcept; + [[nodiscard]] std::string_view redo_name() const noexcept; + [[nodiscard]] Result undo(); + [[nodiscard]] Result redo(); + // Forgets undo and redo, keeping the timeline: after building a new project's initial + // tracks or loading one, so the first undo is the user's first edit. + void clear_history() noexcept; + + // Fresh IDs for objects a caller is about to create (tracks, clips, markers), so the + // caller knows them before executing the command. IDs are never reused, even after undo. + [[nodiscard]] TrackId new_track_id() noexcept; + [[nodiscard]] ClipId new_clip_id() noexcept; + [[nodiscard]] MarkerId new_marker_id() noexcept; + + // Increases on every change (execute, undo, redo), so views know when to refresh. + [[nodiscard]] std::uint64_t revision() const noexcept { return revision_; } + +private: + Timeline timeline_; + std::vector> undo_; + std::vector> redo_; + std::uint64_t revision_ = 0; +}; + +} // namespace oma::timeline diff --git a/libs/timeline/include/oma/timeline/evaluate.hpp b/libs/timeline/include/oma/timeline/evaluate.hpp new file mode 100644 index 0000000..e3a4188 --- /dev/null +++ b/libs/timeline/include/oma/timeline/evaluate.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/rational.hpp" +#include "oma/base/time.hpp" +#include "oma/timeline/ids.hpp" +#include "oma/timeline/model.hpp" + +#include + +// What the timeline shows and plays at one instant (CLAUDE.md §9.1): plain data, testable +// without a GPU. The timeline does not depend on the compositor (§5.2), so this is described in +// timeline terms (media IDs and media times); the playback layer, which owns the decoded +// frames, turns it into the compositor's RenderGraph by resolving each media time to an input. + +namespace oma::timeline { + +struct VideoLayer { + ClipId clip; + MediaId media; + // The media time to show: the clip's exact source time rounded down to the media grid + // (the frame on screen at t is the last one with PTS <= t). + RationalTime media_time; + VideoProperties video; +}; + +struct AudioSource { + ClipId clip; + MediaId media; + RationalTime media_time; // first sample, rounded down to the media grid + RationalTime clip_offset; // time since the clip start, for fades + RationalTime clip_duration; + Rational speed; + AudioProperties audio; +}; + +struct Composition { + RationalTime time; + std::vector video; // bottom first + std::vector audio; +}; + +// Hidden video tracks contribute no layers, muted tracks and clips no audio. Clips on video +// tracks contribute their audio when their media has some. Media times are expressed in the +// media's own timebase (MediaInfo::start). +[[nodiscard]] Result evaluate(const Timeline& timeline, const RationalTime& at); + +} // namespace oma::timeline diff --git a/libs/timeline/include/oma/timeline/ids.hpp b/libs/timeline/include/oma/timeline/ids.hpp new file mode 100644 index 0000000..9042fbb --- /dev/null +++ b/libs/timeline/include/oma/timeline/ids.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include + +// Stable identifiers for timeline objects (CLAUDE.md §10: clips reference media by stable ID, +// never by pointer or path). Zero is never a valid ID. + +namespace oma::timeline { + +template +class Id { +public: + constexpr Id() noexcept = default; + explicit constexpr Id(std::uint64_t value) noexcept : value_(value) {} + + [[nodiscard]] constexpr std::uint64_t value() const noexcept { return value_; } + [[nodiscard]] constexpr bool valid() const noexcept { return value_ != 0; } + + friend constexpr auto operator<=>(Id, Id) noexcept = default; + +private: + std::uint64_t value_ = 0; +}; + +using MediaId = Id; +using ClipId = Id; +using TrackId = Id; +using MarkerId = Id; + +} // namespace oma::timeline + +template +struct std::hash> { + std::size_t operator()(oma::timeline::Id id) const noexcept { + return std::hash{}(id.value()); + } +}; diff --git a/libs/timeline/include/oma/timeline/model.hpp b/libs/timeline/include/oma/timeline/model.hpp new file mode 100644 index 0000000..ee19edc --- /dev/null +++ b/libs/timeline/include/oma/timeline/model.hpp @@ -0,0 +1,211 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/rational.hpp" +#include "oma/base/time.hpp" +#include "oma/timeline/ids.hpp" + +#include +#include +#include +#include +#include + +// The timeline model (CLAUDE.md §10): plain C++, no Qt, GPU or FFmpeg. Read access is public; +// changes go only through commands executed by an Editor (oma/timeline/editor.hpp). + +namespace oma::timeline { + +namespace detail { +class Mutation; +} // namespace detail + +// What the timeline knows about a media item. The library/probe layer fills it; the timeline +// only needs enough to validate references and source ranges. +struct MediaInfo { + MediaId id; + // Available source range in media time, [start, start + duration). Ignored for stills. + RationalTime start; + RationalTime duration; + bool has_video = false; + bool has_audio = false; + bool still = false; // an image: any source range is valid +}; + +// Maps time local to a clip (sequence timebase) to an offset in media time (ADR-0002). +// v0.1 creates constant-speed maps only. Reverse, freeze and ramps need the complementary +// interpolation/domain ADR before they exist (ADR-0002, M5 note); the type is already the one +// the model and the project format use, so adding them needs no structural migration. +class TimeMap { +public: + TimeMap() noexcept = default; + + // Speed > 0: 1 is normal, 1/2 is half speed (clip twice as long), 2 is double speed. + [[nodiscard]] static Result constant(Rational speed); + + [[nodiscard]] Rational speed() const noexcept { return speed_; } + + // Media displacement covered by `local_ticks` of sequence time, exact (no rounding): a time + // in the unit `sequence_timebase * speed`. + [[nodiscard]] Result media_offset(std::int64_t local_ticks, + Rational sequence_timebase) const; + + friend bool operator==(const TimeMap&, const TimeMap&) noexcept = default; + +private: + explicit TimeMap(Rational speed) noexcept : speed_(speed) {} + + Rational speed_ = Rational::literal(1, 1); +}; + +enum class TrackKind : std::uint8_t { + Video, + Audio, + Caption, // in the model from the start; caption clips arrive with v0.2 +}; + +// Visual parameters of a clip. Spatial values are not time, so doubles are fine here; their +// meaning matches the compositor's render graph, which the playback layer builds from them. +enum class Fit : std::uint8_t { + Fit, + Fill, + Stretch, + Native, +}; + +enum class BlendMode : std::uint8_t { + Normal, + Add, + Multiply, + Screen, +}; + +struct Crop { + double left = 0.0; // fractions of the source removed from each edge, in [0, 1) + double top = 0.0; + double right = 0.0; + double bottom = 0.0; + + friend bool operator==(const Crop&, const Crop&) noexcept = default; +}; + +struct Transform { + double offset_x = 0.0; // output pixels from the output center + double offset_y = 0.0; + double scale_x = 1.0; + double scale_y = 1.0; + double rotation = 0.0; // degrees, clockwise + + friend bool operator==(const Transform&, const Transform&) noexcept = default; +}; + +struct VideoProperties { + Fit fit = Fit::Fit; + Crop crop; + Transform transform; + float opacity = 1.0F; + BlendMode blend = BlendMode::Normal; + + friend bool operator==(const VideoProperties&, const VideoProperties&) noexcept = default; +}; + +struct AudioProperties { + float gain = 1.0F; // linear + bool muted = false; + // Fade lengths in timeline time; together they fit in the clip. + RationalTime fade_in; + RationalTime fade_out; + + friend bool operator==(const AudioProperties&, const AudioProperties&) noexcept = default; +}; + +// A clip places a range of a media item on a track. Non-destructive: it only references media. +struct Clip { + ClipId id; + MediaId media; + RationalTime start; // sequence timebase + RationalTime duration; // sequence timebase, > 0 + // Media time shown at the clip's first instant. Exact, in any timebase: trims move it by + // exact amounts, and rounding to a media PTS happens only on evaluation. + RationalTime source_in; + TimeMap time_map; + VideoProperties video; + AudioProperties audio; + + // Exact end: start + duration. + [[nodiscard]] std::int64_t start_ticks() const noexcept { return start.value(); } + [[nodiscard]] std::int64_t end_ticks() const noexcept { + return start.value() + duration.value(); + } +}; + +struct Track { + TrackId id; + TrackKind kind = TrackKind::Video; + std::string name; + bool muted = false; // audio of this track is not mixed + bool hidden = false; // video of this track is not composited + std::vector clips; // sorted by start, never overlapping +}; + +struct Marker { + MarkerId id; + RationalTime time; // sequence timebase + std::string name; +}; + +class Timeline { +public: + // `timebase` is the sequence tick: every timeline position is an integer multiple of it. + // It must hold both the frame grid and audio samples (audio edits are not quantized to + // frames, CLAUDE.md §6); default_timebase() picks one. + [[nodiscard]] static Result create(FrameRate rate, Rational timebase); + [[nodiscard]] static Result create(FrameRate rate, SampleRate audio_rate); + + // 1 / lcm(frame-rate numerator, sample rate): 30000/1001 fps and 48 kHz give 1/240000, + // where one frame is 8008 ticks and one sample 5. + [[nodiscard]] static Result default_timebase(FrameRate rate, SampleRate audio_rate); + + [[nodiscard]] FrameRate frame_rate() const noexcept { return rate_; } + [[nodiscard]] Rational timebase() const noexcept { return timebase_; } + + [[nodiscard]] std::span tracks() const noexcept { return tracks_; } + [[nodiscard]] std::span markers() const noexcept { return markers_; } + [[nodiscard]] std::span media() const noexcept { return media_; } + + [[nodiscard]] const Track* find_track(TrackId id) const noexcept; + [[nodiscard]] const Clip* find_clip(ClipId id) const noexcept; + [[nodiscard]] const Track* track_of(ClipId id) const noexcept; + [[nodiscard]] const MediaInfo* find_media(MediaId id) const noexcept; + [[nodiscard]] const Marker* find_marker(MarkerId id) const noexcept; + // The clip covering `ticks` on a track, if any. + [[nodiscard]] const Clip* clip_at(TrackId track, std::int64_t ticks) const noexcept; + + // A time in sequence ticks. Times off the sequence grid are rejected, never rounded: + // edits land exactly where they were asked to. + [[nodiscard]] Result to_ticks(const RationalTime& t) const; + [[nodiscard]] RationalTime at(std::int64_t ticks) const noexcept; + + // End of the last clip on any track. + [[nodiscard]] RationalTime duration() const noexcept; + + // Checks every invariant: unique IDs, sorted non-overlapping clips with positive + // durations on the sequence grid, existing media of a kind the track accepts, source + // ranges inside the media, fades that fit, valid visual parameters, markers on the grid. + [[nodiscard]] Result validate() const; + +private: + Timeline(FrameRate rate, Rational timebase) noexcept : rate_(rate), timebase_(timebase) {} + + friend class detail::Mutation; + + FrameRate rate_; + Rational timebase_; + std::vector tracks_; // video tracks bottom first, then any order for audio + std::vector markers_; + std::vector media_; + // IDs only grow, even across undo, so an ID is never reused for another object. + std::uint64_t next_id_ = 1; +}; + +} // namespace oma::timeline diff --git a/libs/timeline/module.mk b/libs/timeline/module.mk new file mode 100644 index 0000000..e882fa6 --- /dev/null +++ b/libs/timeline/module.mk @@ -0,0 +1,3 @@ +# libs/timeline — timeline model, edit commands with undo/redo, and per-instant evaluation. +# Depends on base only: no Qt, GPU or FFmpeg (CLAUDE.md §5.2, §10). +$(eval $(call oma_library,timeline,$(wildcard libs/timeline/src/*.cpp),base,,)) diff --git a/libs/timeline/src/commands.cpp b/libs/timeline/src/commands.cpp new file mode 100644 index 0000000..b07386d --- /dev/null +++ b/libs/timeline/src/commands.cpp @@ -0,0 +1,475 @@ +#include "commands.hpp" + +#include "mutation.hpp" + +#include +#include +#include +#include +#include + +namespace oma::timeline::detail { + +namespace { + +void sort_clips(Track& t) { + std::ranges::sort(t.clips, {}, &Clip::start_ticks); +} + +// Whether [start, end) is free on the track, ignoring one clip (the one being replaced). +bool range_free(const Track& t, std::int64_t start, std::int64_t end, ClipId ignore = {}) { + return std::ranges::none_of(t.clips, [&](const Clip& c) { + return c.id != ignore && c.start_ticks() < end && start < c.end_ticks(); + }); +} + +Result clip_end(const Clip& c) { + return add_ticks(c.start.value(), c.duration.value()); +} + +struct ClipLocation { + Track* track = nullptr; + std::size_t index = 0; +}; + +std::optional locate(Mutation& m, ClipId id) { + for (Track& t : m.tracks()) { + const auto it = std::ranges::find(t.clips, id, &Clip::id); + if (it != t.clips.end()) { + return ClipLocation{.track = &t, + .index = static_cast(it - t.clips.begin())}; + } + } + return std::nullopt; +} + +class Transaction final : public Command { +public: + Transaction(std::string name, Steps steps) : name_(std::move(name)), steps_(std::move(steps)) {} + + [[nodiscard]] std::string_view name() const noexcept override { return name_; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + for (std::size_t i = 0; i < steps_.size(); ++i) { + if (auto r = steps_[i]->apply(timeline); !r) { + for (std::size_t j = i; j > 0; --j) { + steps_[j - 1]->revert(timeline); + } + return r; + } + } + return {}; + } + + void revert(Timeline& timeline) noexcept override { + for (auto& step : std::views::reverse(steps_)) { + step->revert(timeline); + } + } + +private: + std::string name_; + Steps steps_; +}; + +class Planned final : public Command { +public: + Planned(std::string name, Planner planner) + : name_(std::move(name)), planner_(std::move(planner)) {} + + [[nodiscard]] std::string_view name() const noexcept override { return name_; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + if (!plan_) { + auto steps = planner_(timeline); + if (!steps) { + return std::unexpected(steps.error()); + } + plan_ = make_transaction(name_, std::move(*steps)); + } + auto r = plan_->apply(timeline); + if (!r) { + plan_.reset(); // the next attempt plans again from the state it finds + } + return r; + } + + void revert(Timeline& timeline) noexcept override { plan_->revert(timeline); } + +private: + std::string name_; + Planner planner_; + std::unique_ptr plan_; +}; + +class InsertClip final : public Command { +public: + InsertClip(TrackId track, Clip clip) : track_(track), clip_(clip) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Insert Clip"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + Track* t = m.track(track_); + if (t == nullptr) { + return error(ErrorCode::InvalidArgument, "unknown track", track_context(track_)); + } + if (!clip_.id.valid() || timeline.find_clip(clip_.id) != nullptr) { + return error(ErrorCode::InvalidArgument, "clip ID is invalid or in use", + clip_context(clip_.id)); + } + auto end = clip_end(clip_); + if (!end) { + return std::unexpected(end.error()); + } + if (!range_free(*t, clip_.start.value(), *end)) { + return error(ErrorCode::InvalidArgument, "the range is occupied", + std::format("{} on {}", clip_context(clip_.id), track_context(track_))); + } + t->clips.push_back(clip_); + sort_clips(*t); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + if (Track* t = m.track(track_); t != nullptr) { + std::erase_if(t->clips, [&](const Clip& c) { return c.id == clip_.id; }); + } + } + +private: + TrackId track_; + Clip clip_; +}; + +class EraseClip final : public Command { +public: + explicit EraseClip(ClipId id) : id_(id) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Remove Clip"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + auto at = locate(m, id_); + if (!at) { + return error(ErrorCode::InvalidArgument, "unknown clip", clip_context(id_)); + } + track_ = at->track->id; + removed_ = at->track->clips[at->index]; + at->track->clips.erase(at->track->clips.begin() + static_cast(at->index)); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + if (Track* t = m.track(track_); t != nullptr) { + t->clips.push_back(removed_); + sort_clips(*t); + } + } + +private: + ClipId id_; + TrackId track_; + Clip removed_; +}; + +class ReplaceClip final : public Command { +public: + explicit ReplaceClip(Clip clip) : clip_(clip) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Change Clip"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + auto at = locate(m, clip_.id); + if (!at) { + return error(ErrorCode::InvalidArgument, "unknown clip", clip_context(clip_.id)); + } + auto end = clip_end(clip_); + if (!end) { + return std::unexpected(end.error()); + } + if (!range_free(*at->track, clip_.start.value(), *end, clip_.id)) { + return error(ErrorCode::InvalidArgument, "the range is occupied", + clip_context(clip_.id)); + } + old_ = std::exchange(at->track->clips[at->index], clip_); + sort_clips(*at->track); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + if (auto at = locate(m, clip_.id)) { + at->track->clips[at->index] = old_; + sort_clips(*at->track); + } + } + +private: + Clip clip_; + Clip old_; +}; + +class ShiftClips final : public Command { +public: + ShiftClips(TrackId track, std::int64_t from, std::int64_t delta) + : track_(track), from_(from), delta_(delta) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Shift Clips"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + Track* t = m.track(track_); + if (t == nullptr) { + return error(ErrorCode::InvalidArgument, "unknown track", track_context(track_)); + } + const auto first = + std::ranges::find_if(t->clips, [&](const Clip& c) { return c.start_ticks() >= from_; }); + if (first == t->clips.end() || delta_ == 0) { + shifted_.clear(); + return {}; + } + // Moving left must not cross time zero or the clips that stay. + const std::int64_t floor = first == t->clips.begin() ? 0 : std::prev(first)->end_ticks(); + for (auto it = first; it != t->clips.end(); ++it) { + auto moved = add_ticks(it->start.value(), delta_); + if (!moved) { + return std::unexpected(moved.error()); + } + if (it == first && *moved < floor) { + return error(ErrorCode::InvalidArgument, + "shift collides with an earlier clip or time zero", + track_context(track_)); + } + if (auto end = add_ticks(*moved, it->duration.value()); !end) { + return std::unexpected(end.error()); + } + } + shifted_.clear(); + for (auto it = first; it != t->clips.end(); ++it) { + it->start = timeline.at(it->start.value() + delta_); + shifted_.push_back(it->id); + } + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + Track* t = m.track(track_); + if (t == nullptr) { + return; + } + for (Clip& c : t->clips) { + if (std::ranges::find(shifted_, c.id) != shifted_.end()) { + c.start = timeline.at(c.start.value() - delta_); + } + } + } + +private: + TrackId track_; + std::int64_t from_; + std::int64_t delta_; + std::vector shifted_; +}; + +class InsertTrack final : public Command { +public: + InsertTrack(Track track, std::size_t index) : track_(std::move(track)), index_(index) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Add Track"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + if (!track_.id.valid() || m.track(track_.id) != nullptr) { + return error(ErrorCode::InvalidArgument, "track ID is invalid or in use", + track_context(track_.id)); + } + if (index_ > m.tracks().size()) { + return error(ErrorCode::OutOfRange, "track index out of range", + track_context(track_.id)); + } + m.tracks().insert(m.tracks().begin() + static_cast(index_), track_); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + std::erase_if(m.tracks(), [&](const Track& t) { return t.id == track_.id; }); + } + +private: + Track track_; + std::size_t index_; +}; + +class EraseTrack final : public Command { +public: + explicit EraseTrack(TrackId id) : id_(id) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Remove Track"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + const auto it = std::ranges::find(m.tracks(), id_, &Track::id); + if (it == m.tracks().end()) { + return error(ErrorCode::InvalidArgument, "unknown track", track_context(id_)); + } + index_ = static_cast(it - m.tracks().begin()); + removed_ = std::move(*it); + m.tracks().erase(it); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + m.tracks().insert(m.tracks().begin() + static_cast(index_), + std::move(removed_)); + } + +private: + TrackId id_; + std::size_t index_ = 0; + Track removed_; +}; + +class SetFlags final : public Command { +public: + SetFlags(TrackId id, bool muted, bool hidden) : id_(id), muted_(muted), hidden_(hidden) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Change Track"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + Mutation m(timeline); + Track* t = m.track(id_); + if (t == nullptr) { + return error(ErrorCode::InvalidArgument, "unknown track", track_context(id_)); + } + old_muted_ = std::exchange(t->muted, muted_); + old_hidden_ = std::exchange(t->hidden, hidden_); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + Mutation m(timeline); + if (Track* t = m.track(id_); t != nullptr) { + t->muted = old_muted_; + t->hidden = old_hidden_; + } + } + +private: + TrackId id_; + bool muted_; + bool hidden_; + bool old_muted_ = false; + bool old_hidden_ = false; +}; + +class InsertMarker final : public Command { +public: + explicit InsertMarker(Marker marker) : marker_(std::move(marker)) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Add Marker"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + if (!marker_.id.valid() || timeline.find_marker(marker_.id) != nullptr) { + return error(ErrorCode::InvalidArgument, "marker ID is invalid or in use", + std::format("marker {}", marker_.id.value())); + } + auto& markers = Mutation(timeline).markers(); + markers.push_back(marker_); + std::ranges::stable_sort(markers, + [](const Marker& a, const Marker& b) { return a.time < b.time; }); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + std::erase_if(Mutation(timeline).markers(), + [&](const Marker& m) { return m.id == marker_.id; }); + } + +private: + Marker marker_; +}; + +class EraseMarker final : public Command { +public: + explicit EraseMarker(MarkerId id) : id_(id) {} + + [[nodiscard]] std::string_view name() const noexcept override { return "Remove Marker"; } + + [[nodiscard]] Result apply(Timeline& timeline) override { + auto& markers = Mutation(timeline).markers(); + const auto it = std::ranges::find(markers, id_, &Marker::id); + if (it == markers.end()) { + return error(ErrorCode::InvalidArgument, "unknown marker", + std::format("marker {}", id_.value())); + } + index_ = static_cast(it - markers.begin()); + removed_ = std::move(*it); + markers.erase(it); + return {}; + } + + void revert(Timeline& timeline) noexcept override { + auto& markers = Mutation(timeline).markers(); + markers.insert(markers.begin() + static_cast(index_), std::move(removed_)); + } + +private: + MarkerId id_; + std::size_t index_ = 0; + Marker removed_; +}; + +} // namespace + +std::unique_ptr make_transaction(std::string name, Steps steps) { + return std::make_unique(std::move(name), std::move(steps)); +} + +std::unique_ptr make_planned(std::string name, Planner planner) { + return std::make_unique(std::move(name), std::move(planner)); +} + +std::unique_ptr insert_clip(TrackId track, Clip clip) { + return std::make_unique(track, clip); +} + +std::unique_ptr erase_clip(ClipId id) { + return std::make_unique(id); +} + +std::unique_ptr replace_clip(Clip clip) { + return std::make_unique(clip); +} + +std::unique_ptr shift_clips(TrackId track, std::int64_t from, std::int64_t delta) { + return std::make_unique(track, from, delta); +} + +std::unique_ptr insert_track(Track track, std::size_t index) { + return std::make_unique(std::move(track), index); +} + +std::unique_ptr erase_track(TrackId id) { + return std::make_unique(id); +} + +std::unique_ptr set_flags(TrackId id, bool muted, bool hidden) { + return std::make_unique(id, muted, hidden); +} + +std::unique_ptr insert_marker(Marker marker) { + return std::make_unique(std::move(marker)); +} + +std::unique_ptr erase_marker(MarkerId id) { + return std::make_unique(id); +} + +} // namespace oma::timeline::detail diff --git a/libs/timeline/src/commands.hpp b/libs/timeline/src/commands.hpp new file mode 100644 index 0000000..ae728c8 --- /dev/null +++ b/libs/timeline/src/commands.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include "oma/timeline/editor.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include +#include + +// Internal building blocks. Primitive commands each change one thing and remember exactly what +// they changed, so revert restores the prior state bit for bit. Editing operations are planned +// commands: on their first apply they read the timeline, plan a list of primitives and apply +// them as one transaction; redo re-applies the same primitives (the state is identical). + +namespace oma::timeline::detail { + +using Steps = std::vector>; +using Planner = std::function(Timeline&)>; + +[[nodiscard]] std::unique_ptr make_transaction(std::string name, Steps steps); +[[nodiscard]] std::unique_ptr make_planned(std::string name, Planner planner); + +[[nodiscard]] std::unique_ptr insert_clip(TrackId track, Clip clip); +[[nodiscard]] std::unique_ptr erase_clip(ClipId id); +// Replaces the clip with the same ID (any field, including its position on the track). +[[nodiscard]] std::unique_ptr replace_clip(Clip clip); +// Moves every clip of the track starting at or after `from` by `delta` ticks. +[[nodiscard]] std::unique_ptr shift_clips(TrackId track, std::int64_t from, + std::int64_t delta); +[[nodiscard]] std::unique_ptr insert_track(Track track, std::size_t index); +[[nodiscard]] std::unique_ptr erase_track(TrackId id); +[[nodiscard]] std::unique_ptr set_flags(TrackId id, bool muted, bool hidden); +[[nodiscard]] std::unique_ptr insert_marker(Marker marker); +[[nodiscard]] std::unique_ptr erase_marker(MarkerId id); + +} // namespace oma::timeline::detail diff --git a/libs/timeline/src/edit.cpp b/libs/timeline/src/edit.cpp new file mode 100644 index 0000000..280e545 --- /dev/null +++ b/libs/timeline/src/edit.cpp @@ -0,0 +1,570 @@ +#include "oma/timeline/edit.hpp" + +#include "commands.hpp" +#include "mutation.hpp" + +#include +#include + +namespace oma::timeline::edit { + +namespace { + +using detail::error; +using detail::Steps; + +Result get_clip(const Timeline& tl, ClipId id) { + const Clip* c = tl.find_clip(id); + if (c == nullptr) { + return error(ErrorCode::InvalidArgument, "unknown clip", detail::clip_context(id)); + } + return c; +} + +Result get_track(const Timeline& tl, TrackId id) { + const Track* t = tl.find_track(id); + if (t == nullptr) { + return error(ErrorCode::InvalidArgument, "unknown track", detail::track_context(id)); + } + return t; +} + +// The clip with its first instant moved to `new_start` and its end kept: the source in moves +// by the same amount of timeline time, mapped exactly through the clip's speed. +Result with_start(const Timeline& tl, Clip c, std::int64_t new_start) { + const std::int64_t end = c.end_ticks(); + if (new_start >= end || new_start < 0) { + return error(ErrorCode::InvalidArgument, "the clip would be empty", + detail::clip_context(c.id)); + } + auto offset = c.time_map.media_offset(new_start - c.start.value(), tl.timebase()); + if (!offset) { + return std::unexpected(offset.error()); + } + auto source_in = detail::add_exact(c.source_in, *offset); + if (!source_in) { + return std::unexpected(source_in.error()); + } + c.source_in = *source_in; + c.start = tl.at(new_start); + c.duration = tl.at(end - new_start); + return c; +} + +Result with_end(const Timeline& tl, Clip c, std::int64_t new_end) { + if (new_end <= c.start.value()) { + return error(ErrorCode::InvalidArgument, "the clip would be empty", + detail::clip_context(c.id)); + } + c.duration = tl.at(new_end - c.start.value()); + return c; +} + +std::int64_t track_end(const Track& t) { + return t.clips.empty() ? 0 : t.clips.back().end_ticks(); +} + +// Empties [from, to) on a track: clips inside are removed, clips crossing an edge are trimmed, +// and a clip covering the whole range is split around it (the tail gets a new ID). +Result clear_range(Timeline& tl, const Track& track, std::int64_t from, std::int64_t to) { + Steps steps; + for (const Clip& c : track.clips) { + const std::int64_t start = c.start_ticks(); + const std::int64_t end = c.end_ticks(); + if (end <= from || start >= to) { + continue; + } + if (start >= from && end <= to) { + steps.push_back(detail::erase_clip(c.id)); + } else if (start < from && end > to) { + auto head = with_end(tl, c, from); + auto tail = with_start(tl, c, to); + if (!head || !tail) { + return std::unexpected(!head ? head.error() : tail.error()); + } + tail->id = ClipId(detail::Mutation(tl).allocate_id()); + steps.push_back(detail::replace_clip(*head)); + steps.push_back(detail::insert_clip(track.id, *tail)); + } else if (start < from) { + auto head = with_end(tl, c, from); + if (!head) { + return std::unexpected(head.error()); + } + steps.push_back(detail::replace_clip(*head)); + } else { + auto tail = with_start(tl, c, to); + if (!tail) { + return std::unexpected(tail.error()); + } + steps.push_back(detail::replace_clip(*tail)); + } + } + return steps; +} + +Result new_clip(const Timeline& tl, ClipId id, std::int64_t start, const ClipSource& source) { + auto duration = tl.to_ticks(source.duration); + if (!duration) { + return std::unexpected(duration.error()); + } + if (*duration <= 0) { + return error(ErrorCode::InvalidArgument, "clip duration must be positive", + detail::clip_context(id)); + } + if (auto end = detail::add_ticks(start, *duration); !end) { + return std::unexpected(end.error()); + } + return Clip{.id = id, + .media = source.media, + .start = tl.at(start), + .duration = tl.at(*duration), + .source_in = source.source_in, + .time_map = source.time_map, + .video = source.video, + .audio = source.audio}; +} + +// A change to a clip plus a shift of the clips after it. Whichever frees space runs first, so +// no intermediate state overlaps: growing shifts first, shrinking changes first. +Steps change_and_shift(Clip changed, TrackId track, std::int64_t shift_from, + std::int64_t shift_by) { + Steps steps; + auto change = detail::replace_clip(changed); + auto shift = detail::shift_clips(track, shift_from, shift_by); + if (shift_by > 0) { + steps.push_back(std::move(shift)); + steps.push_back(std::move(change)); + } else { + steps.push_back(std::move(change)); + steps.push_back(std::move(shift)); + } + return steps; +} + +std::unique_ptr single(std::string name, std::unique_ptr step) { + Steps steps; + steps.push_back(std::move(step)); + return detail::make_transaction(std::move(name), std::move(steps)); +} + +} // namespace + +std::unique_ptr add_track(TrackId id, TrackKind kind, std::string name, + std::optional index) { + return detail::make_planned( + "Add Track", [=, name = std::move(name)](Timeline& tl) -> Result { + Steps steps; + steps.push_back(detail::insert_track(Track{.id = id, + .kind = kind, + .name = name, + .muted = false, + .hidden = false, + .clips = {}}, + index.value_or(tl.tracks().size()))); + return steps; + }); +} + +std::unique_ptr remove_track(TrackId id) { + return single("Remove Track", detail::erase_track(id)); +} + +std::unique_ptr set_track_flags(TrackId id, bool muted, bool hidden) { + return single("Change Track", detail::set_flags(id, muted, hidden)); +} + +std::unique_ptr overwrite(TrackId track, ClipId id, RationalTime at, ClipSource source) { + return detail::make_planned("Overwrite", [=](Timeline& tl) -> Result { + auto t = get_track(tl, track); + auto start = tl.to_ticks(at); + if (!t || !start) { + return std::unexpected(!t ? t.error() : start.error()); + } + auto clip = new_clip(tl, id, *start, source); + if (!clip) { + return std::unexpected(clip.error()); + } + auto steps = clear_range(tl, **t, *start, clip->end_ticks()); + if (!steps) { + return steps; + } + steps->push_back(detail::insert_clip(track, *clip)); + return steps; + }); +} + +std::unique_ptr insert(TrackId track, ClipId id, RationalTime at, ClipSource source) { + return detail::make_planned("Insert", [=](Timeline& tl) -> Result { + auto t = get_track(tl, track); + auto start = tl.to_ticks(at); + if (!t || !start) { + return std::unexpected(!t ? t.error() : start.error()); + } + auto clip = new_clip(tl, id, *start, source); + if (!clip) { + return std::unexpected(clip.error()); + } + const std::int64_t length = clip->duration.value(); + Steps steps; + const Clip* spanning = tl.clip_at(track, *start); + std::optional tail; + if (spanning != nullptr && spanning->start_ticks() < *start) { + // The head ends at the insertion point; the tail keeps its source range and moves + // right by the inserted length. + auto head = with_end(tl, *spanning, *start); + auto moved = with_start(tl, *spanning, *start); + if (!head || !moved) { + return std::unexpected(!head ? head.error() : moved.error()); + } + moved->start = tl.at(*start + length); + moved->id = ClipId(detail::Mutation(tl).allocate_id()); + tail = *moved; + steps.push_back(detail::replace_clip(*head)); + } + steps.push_back(detail::shift_clips(track, *start, length)); + if (tail) { + steps.push_back(detail::insert_clip(track, *tail)); + } + steps.push_back(detail::insert_clip(track, *clip)); + return steps; + }); +} + +std::unique_ptr append(TrackId track, ClipId id, ClipSource source) { + return detail::make_planned("Append", [=](Timeline& tl) -> Result { + auto t = get_track(tl, track); + if (!t) { + return std::unexpected(t.error()); + } + auto clip = new_clip(tl, id, track_end(**t), source); + if (!clip) { + return std::unexpected(clip.error()); + } + Steps steps; + steps.push_back(detail::insert_clip(track, *clip)); + return steps; + }); +} + +std::unique_ptr remove_clip(ClipId id) { + return single("Delete", detail::erase_clip(id)); +} + +std::unique_ptr ripple_delete(ClipId id) { + return detail::make_planned("Ripple Delete", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + if (!c) { + return std::unexpected(c.error()); + } + Steps steps; + steps.push_back(detail::erase_clip(id)); + steps.push_back( + detail::shift_clips(tl.track_of(id)->id, (*c)->end_ticks(), -(*c)->duration.value())); + return steps; + }); +} + +std::unique_ptr ripple_delete_range(TrackId track, RationalTime from, RationalTime to) { + return detail::make_planned("Ripple Delete", [=](Timeline& tl) -> Result { + auto t = get_track(tl, track); + auto a = tl.to_ticks(from); + auto b = tl.to_ticks(to); + if (!t || !a || !b) { + return std::unexpected(!t ? t.error() : (!a ? a.error() : b.error())); + } + if (*b <= *a || *a < 0) { + return error(ErrorCode::InvalidArgument, "empty or negative range", + detail::track_context(track)); + } + auto steps = clear_range(tl, **t, *a, *b); + if (!steps) { + return steps; + } + steps->push_back(detail::shift_clips(track, *b, *a - *b)); + return steps; + }); +} + +std::unique_ptr split(ClipId id, RationalTime at) { + return detail::make_planned("Split", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto point = tl.to_ticks(at); + if (!c || !point) { + return std::unexpected(!c ? c.error() : point.error()); + } + if (*point <= (*c)->start_ticks() || *point >= (*c)->end_ticks()) { + return error(ErrorCode::InvalidArgument, "split point outside the clip", + detail::clip_context(id)); + } + auto head = with_end(tl, **c, *point); + auto tail = with_start(tl, **c, *point); + if (!head || !tail) { + return std::unexpected(!head ? head.error() : tail.error()); + } + tail->id = ClipId(detail::Mutation(tl).allocate_id()); + Steps steps; + steps.push_back(detail::replace_clip(*head)); + steps.push_back(detail::insert_clip(tl.track_of(id)->id, *tail)); + return steps; + }); +} + +std::unique_ptr trim_start(ClipId id, RationalTime new_start, bool ripple) { + return detail::make_planned("Trim", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto start = tl.to_ticks(new_start); + if (!c || !start) { + return std::unexpected(!c ? c.error() : start.error()); + } + auto trimmed = with_start(tl, **c, *start); + if (!trimmed) { + return std::unexpected(trimmed.error()); + } + if (!ripple) { + Steps steps; + steps.push_back(detail::replace_clip(*trimmed)); + return steps; + } + // Ripple: the clip keeps its position; what follows moves by the length change. + const std::int64_t delta = *start - (*c)->start_ticks(); + trimmed->start = (*c)->start; + return change_and_shift(*trimmed, tl.track_of(id)->id, (*c)->end_ticks(), -delta); + }); +} + +std::unique_ptr trim_end(ClipId id, RationalTime new_end, bool ripple) { + return detail::make_planned("Trim", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto end = tl.to_ticks(new_end); + if (!c || !end) { + return std::unexpected(!c ? c.error() : end.error()); + } + auto trimmed = with_end(tl, **c, *end); + if (!trimmed) { + return std::unexpected(trimmed.error()); + } + if (!ripple) { + Steps steps; + steps.push_back(detail::replace_clip(*trimmed)); + return steps; + } + return change_and_shift(*trimmed, tl.track_of(id)->id, (*c)->end_ticks(), + *end - (*c)->end_ticks()); + }); +} + +std::unique_ptr roll(ClipId left, RationalTime edit_point) { + return detail::make_planned("Roll", [=](Timeline& tl) -> Result { + auto l = get_clip(tl, left); + auto point = tl.to_ticks(edit_point); + if (!l || !point) { + return std::unexpected(!l ? l.error() : point.error()); + } + const Track* track = tl.track_of(left); + const Clip* r = tl.clip_at(track->id, (*l)->end_ticks()); + if (r == nullptr || r->start_ticks() != (*l)->end_ticks()) { + return error(ErrorCode::InvalidArgument, "no clip starts at the end of this one", + detail::clip_context(left)); + } + auto new_left = with_end(tl, **l, *point); + auto new_right = with_start(tl, *r, *point); + if (!new_left || !new_right) { + return std::unexpected(!new_left ? new_left.error() : new_right.error()); + } + Steps steps; + auto change_left = detail::replace_clip(*new_left); + auto change_right = detail::replace_clip(*new_right); + // The clip that shrinks goes first so the other one has room. + if (*point > (*l)->end_ticks()) { + steps.push_back(std::move(change_right)); + steps.push_back(std::move(change_left)); + } else { + steps.push_back(std::move(change_left)); + steps.push_back(std::move(change_right)); + } + return steps; + }); +} + +std::unique_ptr slip(ClipId id, RationalTime delta) { + return detail::make_planned("Slip", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto d = tl.to_ticks(delta); + if (!c || !d) { + return std::unexpected(!c ? c.error() : d.error()); + } + auto offset = (*c)->time_map.media_offset(*d, tl.timebase()); + if (!offset) { + return std::unexpected(offset.error()); + } + auto source_in = detail::add_exact((*c)->source_in, *offset); + if (!source_in) { + return std::unexpected(source_in.error()); + } + Clip slipped = **c; + slipped.source_in = *source_in; + Steps steps; + steps.push_back(detail::replace_clip(slipped)); + return steps; + }); +} + +std::unique_ptr slide(ClipId id, RationalTime delta) { + return detail::make_planned("Slide", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto d = tl.to_ticks(delta); + if (!c || !d) { + return std::unexpected(!c ? c.error() : d.error()); + } + const Clip& clip = **c; + const Track* track = tl.track_of(id); + const Clip* prev = + clip.start_ticks() > 0 ? tl.clip_at(track->id, clip.start_ticks() - 1) : nullptr; + const Clip* next = tl.clip_at(track->id, clip.end_ticks()); + if (next != nullptr && next->start_ticks() != clip.end_ticks()) { + next = nullptr; + } + Clip moved = clip; + moved.start = tl.at(clip.start_ticks() + *d); + if (moved.start.value() < 0) { + return error(ErrorCode::InvalidArgument, "slide before time zero", + detail::clip_context(id)); + } + std::unique_ptr change_prev; + std::unique_ptr change_next; + if (prev != nullptr) { + auto p = with_end(tl, *prev, prev->end_ticks() + *d); + if (!p) { + return std::unexpected(p.error()); + } + change_prev = detail::replace_clip(*p); + } + if (next != nullptr) { + auto n = with_start(tl, *next, next->start_ticks() + *d); + if (!n) { + return std::unexpected(n.error()); + } + change_next = detail::replace_clip(*n); + } + // Shrink the neighbour on the side the clip moves toward, move it, grow the other. + Steps steps; + auto& first = *d > 0 ? change_next : change_prev; + auto& last = *d > 0 ? change_prev : change_next; + if (first) { + steps.push_back(std::move(first)); + } + steps.push_back(detail::replace_clip(moved)); + if (last) { + steps.push_back(std::move(last)); + } + return steps; + }); +} + +std::unique_ptr move_clip(ClipId id, TrackId track, RationalTime start) { + return detail::make_planned("Move", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto t = get_track(tl, track); + auto s = tl.to_ticks(start); + if (!c || !t || !s) { + return std::unexpected(!c ? c.error() : (!t ? t.error() : s.error())); + } + if (tl.track_of(id)->kind != (*t)->kind) { + return error(ErrorCode::InvalidArgument, "tracks of different kinds", + detail::clip_context(id)); + } + Clip moved = **c; + moved.start = tl.at(*s); + Steps steps; + steps.push_back(detail::erase_clip(id)); + steps.push_back(detail::insert_clip(track, moved)); + return steps; + }); +} + +std::unique_ptr set_speed(ClipId id, Rational speed, bool ripple) { + return detail::make_planned("Speed", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + auto map = TimeMap::constant(speed); + if (!c || !map) { + return std::unexpected(!c ? c.error() : map.error()); + } + auto length = (*c)->time_map.media_offset((*c)->duration.value(), tl.timebase()); + auto unit = detail::multiply(tl.timebase(), speed); + if (!length || !unit) { + return std::unexpected(!length ? length.error() : unit.error()); + } + // Rounding down keeps the source range inside the media. + auto ticks = rescale(length->value(), length->timebase(), *unit, Rounding::Floor); + if (!ticks) { + return std::unexpected(ticks.error()); + } + if (*ticks <= 0) { + return error(ErrorCode::InvalidArgument, "the clip would be empty", + detail::clip_context(id)); + } + Clip changed = **c; + changed.time_map = *map; + changed.duration = tl.at(*ticks); + if (!ripple) { + Steps steps; + steps.push_back(detail::replace_clip(changed)); + return steps; + } + return change_and_shift(changed, tl.track_of(id)->id, (*c)->end_ticks(), + *ticks - (*c)->duration.value()); + }); +} + +std::unique_ptr set_video(ClipId id, VideoProperties video) { + return detail::make_planned("Video Adjustments", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + if (!c) { + return std::unexpected(c.error()); + } + Clip changed = **c; + changed.video = video; + Steps steps; + steps.push_back(detail::replace_clip(changed)); + return steps; + }); +} + +std::unique_ptr set_audio(ClipId id, AudioProperties audio) { + return detail::make_planned("Audio Adjustments", [=](Timeline& tl) -> Result { + auto c = get_clip(tl, id); + if (!c) { + return std::unexpected(c.error()); + } + Clip changed = **c; + changed.audio = audio; + Steps steps; + steps.push_back(detail::replace_clip(changed)); + return steps; + }); +} + +std::unique_ptr add_marker(MarkerId id, RationalTime at, std::string name) { + return detail::make_planned( + "Add Marker", [=, name = std::move(name)](Timeline& tl) -> Result { + auto ticks = tl.to_ticks(at); + if (!ticks) { + return std::unexpected(ticks.error()); + } + Steps steps; + steps.push_back( + detail::insert_marker(Marker{.id = id, .time = tl.at(*ticks), .name = name})); + return steps; + }); +} + +std::unique_ptr remove_marker(MarkerId id) { + return single("Remove Marker", detail::erase_marker(id)); +} + +std::unique_ptr transaction(std::string name, + std::vector> commands) { + return detail::make_transaction(std::move(name), std::move(commands)); +} + +} // namespace oma::timeline::edit diff --git a/libs/timeline/src/editor.cpp b/libs/timeline/src/editor.cpp new file mode 100644 index 0000000..51249f3 --- /dev/null +++ b/libs/timeline/src/editor.cpp @@ -0,0 +1,96 @@ +#include "oma/timeline/editor.hpp" + +#include "mutation.hpp" + +#include + +namespace oma::timeline { + +Result Editor::execute(std::unique_ptr command) { + if (!command) { + return detail::error(ErrorCode::InvalidArgument, "no command"); + } + if (auto r = command->apply(timeline_); !r) { + return r; + } + // Commands check their local preconditions (overlaps, edges); the timeline-wide rules + // (media kinds, source ranges, properties) are checked once here. Rejecting an edit is a + // normal outcome reported to the caller, not a fault, so it is not logged. + if (auto valid = timeline_.validate(); !valid) { + command->revert(timeline_); + return valid; + } + undo_.push_back(std::move(command)); + redo_.clear(); + ++revision_; + return {}; +} + +Result Editor::add_media(MediaInfo media) { + if (!media.id.valid() || timeline_.find_media(media.id) != nullptr) { + return detail::error(ErrorCode::InvalidArgument, "media ID is invalid or in use", + std::format("media {}", media.id.value())); + } + if (!media.still && + (media.duration.value() < 0 || media.start.timebase() != media.duration.timebase())) { + return detail::error(ErrorCode::InvalidArgument, "invalid media range", + std::format("media {}", media.id.value())); + } + detail::Mutation(timeline_).media().push_back(media); + ++revision_; + return {}; +} + +std::string_view Editor::undo_name() const noexcept { + return undo_.empty() ? std::string_view{} : undo_.back()->name(); +} + +std::string_view Editor::redo_name() const noexcept { + return redo_.empty() ? std::string_view{} : redo_.back()->name(); +} + +Result Editor::undo() { + if (undo_.empty()) { + return detail::error(ErrorCode::InvalidArgument, "nothing to undo"); + } + undo_.back()->revert(timeline_); + redo_.push_back(std::move(undo_.back())); + undo_.pop_back(); + ++revision_; + return {}; +} + +Result Editor::redo() { + if (redo_.empty()) { + return detail::error(ErrorCode::InvalidArgument, "nothing to redo"); + } + // The state is the one the command first applied to, so this succeeds unless the command + // is not deterministic; then it stays on the redo stack and the timeline is unchanged. + if (auto r = redo_.back()->apply(timeline_); !r) { + return r; + } + undo_.push_back(std::move(redo_.back())); + redo_.pop_back(); + ++revision_; + return {}; +} + +void Editor::clear_history() noexcept { + undo_.clear(); + redo_.clear(); + ++revision_; +} + +TrackId Editor::new_track_id() noexcept { + return TrackId(detail::Mutation(timeline_).allocate_id()); +} + +ClipId Editor::new_clip_id() noexcept { + return ClipId(detail::Mutation(timeline_).allocate_id()); +} + +MarkerId Editor::new_marker_id() noexcept { + return MarkerId(detail::Mutation(timeline_).allocate_id()); +} + +} // namespace oma::timeline diff --git a/libs/timeline/src/evaluate.cpp b/libs/timeline/src/evaluate.cpp new file mode 100644 index 0000000..528b6df --- /dev/null +++ b/libs/timeline/src/evaluate.cpp @@ -0,0 +1,64 @@ +#include "oma/timeline/evaluate.hpp" + +#include "mutation.hpp" + +namespace oma::timeline { + +namespace { + +// The media time at `ticks` inside a clip, rounded down to the media's own timebase. +Result media_time(const Timeline& tl, const Clip& c, const MediaInfo& m, + std::int64_t ticks) { + auto offset = c.time_map.media_offset(ticks - c.start_ticks(), tl.timebase()); + if (!offset) { + return std::unexpected(offset.error()); + } + auto exact = detail::add_exact(c.source_in, *offset); + if (!exact) { + return std::unexpected(exact.error()); + } + return exact->rescaled(m.start.timebase(), Rounding::Floor); +} + +} // namespace + +Result evaluate(const Timeline& timeline, const RationalTime& at) { + auto ticks = timeline.to_ticks(at); + if (!ticks) { + return std::unexpected(ticks.error()); + } + Composition out{.time = timeline.at(*ticks), .video = {}, .audio = {}}; + for (const Track& track : timeline.tracks()) { + const Clip* c = timeline.clip_at(track.id, *ticks); + if (c == nullptr) { + continue; + } + const MediaInfo* m = timeline.find_media(c->media); + if (m == nullptr) { + return detail::error(ErrorCode::InvalidData, "clip references unknown media", + detail::clip_context(c->id)); + } + auto t = media_time(timeline, *c, *m, *ticks); + if (!t) { + return std::unexpected(t.error()); + } + if (track.kind == TrackKind::Video && !track.hidden) { + out.video.push_back( + VideoLayer{.clip = c->id, .media = c->media, .media_time = *t, .video = c->video}); + } + const bool plays_audio = + track.kind == TrackKind::Audio || (track.kind == TrackKind::Video && m->has_audio); + if (plays_audio && !track.muted && !c->audio.muted) { + out.audio.push_back(AudioSource{.clip = c->id, + .media = c->media, + .media_time = *t, + .clip_offset = timeline.at(*ticks - c->start_ticks()), + .clip_duration = c->duration, + .speed = c->time_map.speed(), + .audio = c->audio}); + } + } + return out; +} + +} // namespace oma::timeline diff --git a/libs/timeline/src/model.cpp b/libs/timeline/src/model.cpp new file mode 100644 index 0000000..a71c48c --- /dev/null +++ b/libs/timeline/src/model.cpp @@ -0,0 +1,278 @@ +#include "oma/timeline/model.hpp" + +#include "mutation.hpp" + +#include +#include +#include +#include +#include + +namespace oma::timeline { + +using detail::error; + +Result TimeMap::constant(Rational speed) { + if (!speed.is_positive()) { + return error(ErrorCode::InvalidArgument, "speed must be positive", + std::format("{}/{}", speed.num(), speed.den())); + } + return TimeMap(speed); +} + +Result TimeMap::media_offset(std::int64_t local_ticks, + Rational sequence_timebase) const { + auto unit = detail::multiply(sequence_timebase, speed_); + if (!unit) { + return std::unexpected(unit.error()); + } + return RationalTime::make(local_ticks, *unit); +} + +Result Timeline::create(FrameRate rate, Rational timebase) { + if (!timebase.is_positive()) { + return error(ErrorCode::InvalidArgument, "sequence timebase must be positive"); + } + // Every frame boundary must be a whole number of ticks. + auto frame = rescale(1, rate.frame_duration(), timebase, Rounding::Floor); + if (!frame || *frame <= 0 || + *RationalTime::make(*frame, timebase) != *RationalTime::make(1, rate.frame_duration())) { + return error(ErrorCode::InvalidArgument, + "the frame duration is not a whole number of sequence ticks", + std::format("{}/{} fps, timebase {}/{}", rate.fps().num(), rate.fps().den(), + timebase.num(), timebase.den())); + } + return Timeline(rate, timebase); +} + +Result Timeline::create(FrameRate rate, SampleRate audio_rate) { + auto timebase = default_timebase(rate, audio_rate); + if (!timebase) { + return std::unexpected(timebase.error()); + } + return create(rate, *timebase); +} + +Result Timeline::default_timebase(FrameRate rate, SampleRate audio_rate) { + // A frame lasts den/num seconds, a sample 1/hz: 1/lcm(num, hz) divides both. + const std::int64_t num = rate.fps().num(); + const std::int64_t hz = audio_rate.hz(); + const std::int64_t g = std::gcd(num, hz); + std::int64_t lcm = 0; + if (__builtin_mul_overflow(num / g, hz, &lcm)) { + return error(ErrorCode::Overflow, "no common sequence timebase"); + } + return Rational::make(1, lcm); +} + +const Track* Timeline::find_track(TrackId id) const noexcept { + const auto it = std::ranges::find(tracks_, id, &Track::id); + return it == tracks_.end() ? nullptr : &*it; +} + +const Clip* Timeline::find_clip(ClipId id) const noexcept { + for (const Track& t : tracks_) { + const auto it = std::ranges::find(t.clips, id, &Clip::id); + if (it != t.clips.end()) { + return &*it; + } + } + return nullptr; +} + +const Track* Timeline::track_of(ClipId id) const noexcept { + for (const Track& t : tracks_) { + if (std::ranges::find(t.clips, id, &Clip::id) != t.clips.end()) { + return &t; + } + } + return nullptr; +} + +const MediaInfo* Timeline::find_media(MediaId id) const noexcept { + const auto it = std::ranges::find(media_, id, &MediaInfo::id); + return it == media_.end() ? nullptr : &*it; +} + +const Marker* Timeline::find_marker(MarkerId id) const noexcept { + const auto it = std::ranges::find(markers_, id, &Marker::id); + return it == markers_.end() ? nullptr : &*it; +} + +const Clip* Timeline::clip_at(TrackId track, std::int64_t ticks) const noexcept { + const Track* t = find_track(track); + if (t == nullptr) { + return nullptr; + } + // Clips are sorted by start: the candidate is the last one starting at or before `ticks`. + const auto it = std::ranges::upper_bound(t->clips, ticks, {}, &Clip::start_ticks); + if (it == t->clips.begin()) { + return nullptr; + } + const Clip& c = *std::prev(it); + return ticks < c.end_ticks() ? &c : nullptr; +} + +Result Timeline::to_ticks(const RationalTime& t) const { + auto ticks = rescale(t.value(), t.timebase(), timebase_, Rounding::Floor); + if (!ticks) { + return std::unexpected(ticks.error()); + } + if (at(*ticks) != t) { + return error(ErrorCode::InvalidArgument, "time is not on the sequence grid", + std::format("{} @ {}/{}", t.value(), t.timebase().num(), t.timebase().den())); + } + return *ticks; +} + +RationalTime Timeline::at(std::int64_t ticks) const noexcept { + return *RationalTime::make(ticks, timebase_); +} + +RationalTime Timeline::duration() const noexcept { + std::int64_t end = 0; + for (const Track& t : tracks_) { + if (!t.clips.empty()) { + end = std::max(end, t.clips.back().end_ticks()); + } + } + return at(end); +} + +namespace { + +bool in_unit(double v) { + return std::isfinite(v) && v >= 0.0 && v < 1.0; +} + +Result validate_properties(const Clip& c) { + const VideoProperties& v = c.video; + const bool crop_ok = in_unit(v.crop.left) && in_unit(v.crop.right) && in_unit(v.crop.top) && + in_unit(v.crop.bottom) && v.crop.left + v.crop.right < 1.0 && + v.crop.top + v.crop.bottom < 1.0; + const Transform& tr = v.transform; + const bool transform_ok = std::isfinite(tr.offset_x) && std::isfinite(tr.offset_y) && + std::isfinite(tr.rotation) && std::isfinite(tr.scale_x) && + std::isfinite(tr.scale_y) && tr.scale_x != 0.0 && tr.scale_y != 0.0; + const bool opacity_ok = std::isfinite(v.opacity) && v.opacity >= 0.0F && v.opacity <= 1.0F; + if (!crop_ok || !transform_ok || !opacity_ok) { + return error(ErrorCode::InvalidData, "invalid video properties", + detail::clip_context(c.id)); + } + const AudioProperties& a = c.audio; + if (!std::isfinite(a.gain) || a.gain < 0.0F) { + return error(ErrorCode::InvalidData, "invalid audio gain", detail::clip_context(c.id)); + } + const RationalTime zero; + auto fades = detail::add_exact(a.fade_in, a.fade_out); + if (a.fade_in < zero || a.fade_out < zero || !fades || c.duration < *fades) { + return error(ErrorCode::InvalidData, "fades do not fit in the clip", + detail::clip_context(c.id)); + } + return {}; +} + +Result validate_source(const Clip& c, const MediaInfo& m, Rational timebase) { + if (m.still) { + return {}; + } + auto end = detail::source_end(c, timebase); + if (!end) { + return std::unexpected(end.error()); + } + auto media_end = m.start.plus(m.duration); + if (!media_end) { + return std::unexpected(media_end.error()); + } + if (c.source_in < m.start || *media_end < *end) { + return error(ErrorCode::InvalidData, "source range outside the media", + detail::clip_context(c.id)); + } + return {}; +} + +bool media_fits_track(const MediaInfo& m, TrackKind kind) { + switch (kind) { + case TrackKind::Video: + return m.has_video; + case TrackKind::Audio: + return m.has_audio; + case TrackKind::Caption: + return false; // caption clips are not media clips (v0.2) + } + return false; +} + +} // namespace + +Result Timeline::validate() const { + std::unordered_set media_ids; + for (const MediaInfo& m : media_) { + if (!m.id.valid() || !media_ids.insert(m.id).second) { + return error(ErrorCode::InvalidData, "invalid or duplicate media ID", + std::format("media {}", m.id.value())); + } + if (!m.still && (m.duration.value() < 0 || m.start.timebase() != m.duration.timebase())) { + return error(ErrorCode::InvalidData, "invalid media range", + std::format("media {}", m.id.value())); + } + } + + std::unordered_set track_ids; + std::unordered_set clip_ids; + for (const Track& t : tracks_) { + if (!t.id.valid() || !track_ids.insert(t.id).second) { + return error(ErrorCode::InvalidData, "invalid or duplicate track ID", + detail::track_context(t.id)); + } + std::int64_t previous_end = 0; + for (const Clip& c : t.clips) { + if (!c.id.valid() || !clip_ids.insert(c.id).second) { + return error(ErrorCode::InvalidData, "invalid or duplicate clip ID", + detail::clip_context(c.id)); + } + if (c.start.timebase() != timebase_ || c.duration.timebase() != timebase_) { + return error(ErrorCode::InvalidData, "clip off the sequence timebase", + detail::clip_context(c.id)); + } + if (c.duration.value() <= 0 || c.start.value() < previous_end) { + return error(ErrorCode::InvalidData, "empty, overlapping or unsorted clip", + std::format("{} on {}", detail::clip_context(c.id), + detail::track_context(t.id))); + } + auto end = detail::add_ticks(c.start.value(), c.duration.value()); + if (!end) { + return std::unexpected(end.error()); + } + previous_end = *end; + const MediaInfo* m = find_media(c.media); + if (m == nullptr) { + return error(ErrorCode::InvalidData, "clip references unknown media", + detail::clip_context(c.id)); + } + if (!media_fits_track(*m, t.kind)) { + return error(ErrorCode::InvalidData, "media kind does not fit the track", + std::format("{} on {}", detail::clip_context(c.id), + detail::track_context(t.id))); + } + if (auto r = validate_source(c, *m, timebase_); !r) { + return r; + } + if (auto r = validate_properties(c); !r) { + return r; + } + } + } + + std::unordered_set marker_ids; + for (const Marker& m : markers_) { + if (!m.id.valid() || !marker_ids.insert(m.id).second || m.time.timebase() != timebase_ || + m.time.value() < 0) { + return error(ErrorCode::InvalidData, "invalid marker", + std::format("marker {}", m.id.value())); + } + } + return {}; +} + +} // namespace oma::timeline diff --git a/libs/timeline/src/mutation.hpp b/libs/timeline/src/mutation.hpp new file mode 100644 index 0000000..7d5e90d --- /dev/null +++ b/libs/timeline/src/mutation.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/rational.hpp" +#include "oma/base/time.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include + +// Internal write access to a Timeline. Only commands (and the Editor's ID allocation) use it, +// which keeps Editor::execute the single entry point for edits. + +namespace oma::timeline::detail { + +class Mutation { +public: + explicit Mutation(Timeline& timeline) noexcept : t_(timeline) {} + + [[nodiscard]] std::vector& tracks() noexcept { return t_.tracks_; } + [[nodiscard]] std::vector& markers() noexcept { return t_.markers_; } + [[nodiscard]] std::vector& media() noexcept { return t_.media_; } + [[nodiscard]] std::uint64_t allocate_id() noexcept { return t_.next_id_++; } + + [[nodiscard]] Track* track(TrackId id) noexcept; + +private: + Timeline& t_; +}; + +// Exact time arithmetic for source positions, which may use any timebase. + +// a * b, reduced; Overflow when the reduced terms do not fit in int64. +[[nodiscard]] Result multiply(Rational a, Rational b); +// The largest unit both timebases are integer multiples of (gcd of two positive rationals). +[[nodiscard]] Result common_unit(Rational a, Rational b); +// a + b exactly, in their common unit. +[[nodiscard]] Result add_exact(const RationalTime& a, const RationalTime& b); +// Overflow-checked integer addition for sequence ticks. +[[nodiscard]] Result add_ticks(std::int64_t a, std::int64_t b); + +// The exact media time reached at the end of a clip: source_in + map(duration). +[[nodiscard]] Result source_end(const Clip& clip, Rational sequence_timebase); + +[[nodiscard]] std::unexpected error(ErrorCode code, std::string message, + std::string context = {}); +[[nodiscard]] std::string clip_context(ClipId id); +[[nodiscard]] std::string track_context(TrackId id); + +} // namespace oma::timeline::detail diff --git a/libs/timeline/src/time_math.cpp b/libs/timeline/src/time_math.cpp new file mode 100644 index 0000000..b72c50f --- /dev/null +++ b/libs/timeline/src/time_math.cpp @@ -0,0 +1,109 @@ +#include "mutation.hpp" + +#include +#include +#include + +namespace oma::timeline::detail { + +namespace { + +__extension__ using Int128 = __int128; + +Int128 gcd128(Int128 a, Int128 b) { + a = a < 0 ? -a : a; + b = b < 0 ? -b : b; + while (b != 0) { + const Int128 r = a % b; + a = b; + b = r; + } + return a; +} + +bool fits(Int128 v) { + return v >= std::numeric_limits::min() + 1 && + v <= std::numeric_limits::max(); +} + +Result reduced(Int128 num, Int128 den) { + const Int128 g = gcd128(num, den); + if (g != 0) { + num /= g; + den /= g; + } + if (!fits(num) || !fits(den)) { + return error(ErrorCode::Overflow, "rational term exceeds 64 bits"); + } + return Rational::make(static_cast(num), static_cast(den)); +} + +} // namespace + +Track* Mutation::track(TrackId id) noexcept { + const auto it = std::ranges::find(t_.tracks_, id, &Track::id); + return it == t_.tracks_.end() ? nullptr : &*it; +} + +Result multiply(Rational a, Rational b) { + return reduced(static_cast(a.num()) * b.num(), static_cast(a.den()) * b.den()); +} + +Result common_unit(Rational a, Rational b) { + // gcd(p/q, r/s) = gcd(p*s, r*q) / (q*s) for positive fractions. + const Int128 num = + gcd128(static_cast(a.num()) * b.den(), static_cast(b.num()) * a.den()); + return reduced(num, static_cast(a.den()) * b.den()); +} + +Result add_exact(const RationalTime& a, const RationalTime& b) { + if (a.timebase() == b.timebase()) { + return a.plus(b); + } + auto unit = common_unit(a.timebase(), b.timebase()); + if (!unit) { + return std::unexpected(unit.error()); + } + // Both timebases are integer multiples of the unit, so these conversions are exact. + auto va = rescale(a.value(), a.timebase(), *unit, Rounding::Floor); + auto vb = rescale(b.value(), b.timebase(), *unit, Rounding::Floor); + if (!va || !vb) { + return std::unexpected(!va ? va.error() : vb.error()); + } + auto sum = add_ticks(*va, *vb); + if (!sum) { + return std::unexpected(sum.error()); + } + return RationalTime::make(*sum, *unit); +} + +Result add_ticks(std::int64_t a, std::int64_t b) { + std::int64_t out = 0; + if (__builtin_add_overflow(a, b, &out)) { + return error(ErrorCode::Overflow, "timeline position overflow", + std::format("{} + {}", a, b)); + } + return out; +} + +Result source_end(const Clip& clip, Rational sequence_timebase) { + auto length = clip.time_map.media_offset(clip.duration.value(), sequence_timebase); + if (!length) { + return std::unexpected(length.error()); + } + return add_exact(clip.source_in, *length); +} + +std::unexpected error(ErrorCode code, std::string message, std::string context) { + return make_error(code, Category::Timeline, std::move(message), std::move(context)); +} + +std::string clip_context(ClipId id) { + return std::format("clip {}", id.value()); +} + +std::string track_context(TrackId id) { + return std::format("track {}", id.value()); +} + +} // namespace oma::timeline::detail diff --git a/tests/timeline/fixture.hpp b/tests/timeline/fixture.hpp new file mode 100644 index 0000000..227e4f1 --- /dev/null +++ b/tests/timeline/fixture.hpp @@ -0,0 +1,108 @@ +#pragma once + +// Shared helpers for the timeline tests. The sequence runs at 30 fps with a 1/30 timebase, so +// one tick is one frame and layouts read in frames. + +#include "oma/timeline/edit.hpp" +#include "oma/timeline/editor.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include + +namespace timeline_test { + +using namespace oma::timeline; + +inline oma::Rational q(std::int64_t num, std::int64_t den) { + return oma::Rational::make(num, den).value(); +} + +// `frames` of sequence time (1/30 s each). +inline oma::RationalTime f(std::int64_t frames) { + return oma::RationalTime::make(frames, q(1, 30)).value(); +} + +// Media time in a 90 kHz timebase, given in 30 fps frames (3000 ticks each). +inline oma::RationalTime mf(std::int64_t frames) { + return oma::RationalTime::make(frames * 3000, q(1, 90000)).value(); +} + +inline const MediaId kCamera{1}; // video + audio, 300 frames +inline const MediaId kMusic{2}; // audio only, 60 s at 48 kHz +inline const MediaId kPicture{3}; // still image + +struct Fixture { + Editor editor; + TrackId video; + TrackId overlay; + TrackId audio; +}; + +inline Fixture make_fixture() { + Fixture fx{Editor(Timeline::create(oma::frame_rates::k30, q(1, 30)).value()), {}, {}, {}}; + Editor& ed = fx.editor; + (void)ed.add_media(MediaInfo{.id = kCamera, + .start = mf(0), + .duration = mf(300), + .has_video = true, + .has_audio = true, + .still = false}); + (void)ed.add_media( + MediaInfo{.id = kMusic, + .start = oma::RationalTime::make(0, q(1, 48000)).value(), + .duration = oma::RationalTime::make(60 * 48000, q(1, 48000)).value(), + .has_video = false, + .has_audio = true, + .still = false}); + (void)ed.add_media(MediaInfo{.id = kPicture, + .start = {}, + .duration = {}, + .has_video = true, + .has_audio = false, + .still = true}); + fx.video = ed.new_track_id(); + fx.overlay = ed.new_track_id(); + fx.audio = ed.new_track_id(); + (void)ed.execute(edit::add_track(fx.video, TrackKind::Video, "V1")); + (void)ed.execute(edit::add_track(fx.overlay, TrackKind::Video, "V2")); + (void)ed.execute(edit::add_track(fx.audio, TrackKind::Audio, "A1")); + return fx; +} + +// Camera media from frame `in`, `duration` frames long. +inline edit::ClipSource camera(std::int64_t in, std::int64_t duration) { + return edit::ClipSource{.media = kCamera, + .source_in = mf(in), + .duration = f(duration), + .time_map = {}, + .video = {}, + .audio = {}}; +} + +// Places a camera clip with overwrite and returns its ID (invalid if the edit failed). +inline ClipId place(Fixture& fx, TrackId track, std::int64_t at, std::int64_t in, + std::int64_t duration) { + const ClipId id = fx.editor.new_clip_id(); + return fx.editor.execute(edit::overwrite(track, id, f(at), camera(in, duration))) ? id + : ClipId{}; +} + +// "start+duration@source" per clip, in frames; source shown in 30 fps frames, '~' if inexact. +inline std::string layout(const Timeline& tl, TrackId track) { + std::string out; + for (const Clip& c : tl.find_track(track)->clips) { + const auto src = c.source_in.rescaled(q(1, 30), oma::Rounding::Floor).value(); + const bool exact = src == c.source_in; + out += std::format("{}{}+{}@{}{}", out.empty() ? "" : " ", c.start.value(), + c.duration.value(), src.value(), exact ? "" : "~"); + } + return out; +} + +inline std::string layout(const Fixture& fx) { + return layout(fx.editor.timeline(), fx.video); +} + +} // namespace timeline_test diff --git a/tests/timeline/main.cpp b/tests/timeline/main.cpp new file mode 100644 index 0000000..e379b81 --- /dev/null +++ b/tests/timeline/main.cpp @@ -0,0 +1,13 @@ +#include "oma_test.hpp" + +void run_model_tests(); +void run_edit_tests(); +void run_evaluate_tests(); + +int main(int argc, char* argv[]) { + cest_init(argc, argv); + run_model_tests(); + run_edit_tests(); + run_evaluate_tests(); + return cest_result(); +} diff --git a/tests/timeline/module.mk b/tests/timeline/module.mk new file mode 100644 index 0000000..cc5ff74 --- /dev/null +++ b/tests/timeline/module.mk @@ -0,0 +1,3 @@ +# libs/timeline tests: model invariants, every edit with undo/redo, and per-instant evaluation. +# No GPU, media files or Qt. +$(eval $(call oma_test,timeline_tests,$(wildcard tests/timeline/*.cpp),timeline base)) diff --git a/tests/timeline/test_edit.cpp b/tests/timeline/test_edit.cpp new file mode 100644 index 0000000..4881545 --- /dev/null +++ b/tests/timeline/test_edit.cpp @@ -0,0 +1,448 @@ +#include "fixture.hpp" + +#include +#include + +#include "oma_test.hpp" + +using namespace timeline_test; + +namespace { + +bool ok(Fixture& fx, std::unique_ptr command) { + return fx.editor.execute(std::move(command)).has_value(); +} + +// Three adjacent clips: 0+10@0 10+10@50 20+10@100. +Fixture three_clips() { + Fixture fx = make_fixture(); + place(fx, fx.video, 0, 0, 10); + place(fx, fx.video, 10, 50, 10); + place(fx, fx.video, 20, 100, 10); + return fx; +} + +ClipId clip_at(const Fixture& fx, std::int64_t frame) { + const Clip* c = fx.editor.timeline().clip_at(fx.video, frame); + return c != nullptr ? c->id : ClipId{}; +} + +// Applies every command, recording the layout after each, then undoes and redoes all of them +// checking that every intermediate layout comes back exactly. +bool undo_redo_round_trip(Fixture& fx, std::vector> commands) { + std::vector states{layout(fx)}; + for (auto& c : commands) { + if (!fx.editor.execute(std::move(c))) { + return false; + } + states.push_back(layout(fx)); + } + for (std::size_t i = states.size() - 1; i > 0; --i) { + if (!fx.editor.undo() || layout(fx) != states[i - 1]) { + return false; + } + } + for (std::size_t i = 1; i < states.size(); ++i) { + if (!fx.editor.redo() || layout(fx) != states[i]) { + return false; + } + } + return !fx.editor.can_redo(); +} + +std::unique_ptr overwrite_music(TrackId track, ClipId id, std::int64_t at) { + return edit::overwrite( + track, id, f(at), + edit::ClipSource{.media = kMusic, + .source_in = oma::RationalTime::make(0, q(1, 48000)).value(), + .duration = f(10), + .time_map = {}, + .video = {}, + .audio = {}}); +} + +MediaInfo camera_media() { + return MediaInfo{.id = kCamera, + .start = mf(0), + .duration = mf(30), + .has_video = true, + .has_audio = false, + .still = false}; +} + +} // namespace + +void run_edit_tests() { + describe("timeline edits: placing clips", { + it("appends clips back to back", { + auto fx = make_fixture(); + const TrackId v = fx.video; + expect(ok(fx, edit::append(v, fx.editor.new_clip_id(), camera(0, 10)))).toBeTruthy(); + expect(ok(fx, edit::append(v, fx.editor.new_clip_id(), camera(30, 5)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+5@30"); + }); + + it("overwrites the middle of a clip by splitting it", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 30); + expect(place(fx, fx.video, 10, 100, 5).valid()).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+5@100 15+15@15"); + }); + + it("overwrites across an edit point by trimming both sides", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 10); + place(fx, fx.video, 10, 50, 10); + place(fx, fx.video, 5, 200, 10); + expect(layout(fx)).toEqual("0+5@0 5+10@200 15+5@55"); + }); + + it("inserts inside a clip by splitting it and pushing the rest", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 30); + place(fx, fx.video, 40, 200, 5); + const ClipId id = fx.editor.new_clip_id(); + expect(ok(fx, edit::insert(fx.video, id, f(10), camera(100, 5)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+5@100 15+20@10 45+5@200"); + }); + + it("inserts at an edit point without splitting", { + auto fx = three_clips(); + expect(ok(fx, edit::insert(fx.video, fx.editor.new_clip_id(), f(10), camera(200, 5)))) + .toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+5@200 15+10@50 25+10@100"); + }); + + it("rejects media of the wrong kind and leaves the timeline unchanged", { + auto fx = three_clips(); + const auto before = layout(fx); + const auto revision = fx.editor.revision(); + expect(ok(fx, overwrite_music(fx.video, fx.editor.new_clip_id(), 0))).toBeFalsy(); + expect(layout(fx)).toEqual(before); + expect(fx.editor.revision()).toBe(revision); + expect(std::string(fx.editor.undo_name())).toEqual("Overwrite"); + }); + + it("accepts audio-only media on an audio track", { + auto fx = make_fixture(); + expect(ok(fx, overwrite_music(fx.audio, fx.editor.new_clip_id(), 0))).toBeTruthy(); + expect(fx.editor.timeline().find_track(fx.audio)->clips.size()).toBe(1U); + }); + + it("rejects a source range beyond the media", { + auto fx = make_fixture(); + expect(place(fx, fx.video, 0, 295, 10).valid()).toBeFalsy(); + expect(place(fx, fx.video, 0, 290, 10).valid()).toBeTruthy(); + }); + + it("rejects reusing a clip ID", { + auto fx = make_fixture(); + const ClipId id = fx.editor.new_clip_id(); + expect(ok(fx, edit::overwrite(fx.video, id, f(0), camera(0, 5)))).toBeTruthy(); + expect(ok(fx, edit::overwrite(fx.overlay, id, f(0), camera(0, 5)))).toBeFalsy(); + }); + + it("rejects times off the sequence grid", { + auto fx = make_fixture(); + const auto half_frame = oma::RationalTime::make(1, q(1, 60)).value(); + expect(ok(fx, + edit::overwrite(fx.video, fx.editor.new_clip_id(), half_frame, camera(0, 5)))) + .toBeFalsy(); + }); + }); + + describe("timeline edits: deleting and splitting", { + it("deletes a clip and leaves a gap", { + auto fx = three_clips(); + expect(ok(fx, edit::remove_clip(clip_at(fx, 10)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 20+10@100"); + }); + + it("ripple deletes a clip and closes the gap", { + auto fx = three_clips(); + expect(ok(fx, edit::ripple_delete(clip_at(fx, 10)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+10@100"); + }); + + it("ripple deletes a range across clips", { + auto fx = three_clips(); + expect(ok(fx, edit::ripple_delete_range(fx.video, f(5), f(25)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+5@0 5+5@105"); + }); + + it("ripple deletes a range inside one clip", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 30); + expect(ok(fx, edit::ripple_delete_range(fx.video, f(10), f(20)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+10@20"); + }); + + it("splits a clip and gives the right part a new ID", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 0, 30); + expect(ok(fx, edit::split(id, f(12)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+12@0 12+18@12"); + expect(clip_at(fx, 0) == id).toBeTruthy(); + expect(clip_at(fx, 12) != id && clip_at(fx, 12).valid()).toBeTruthy(); + }); + + it("refuses to split at a clip edge", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 0, 30); + expect(ok(fx, edit::split(id, f(0)))).toBeFalsy(); + expect(ok(fx, edit::split(id, f(30)))).toBeFalsy(); + }); + }); + + describe("timeline edits: trimming", { + it("trims the start in place, moving the source in", { + auto fx = three_clips(); + expect(ok(fx, edit::remove_clip(clip_at(fx, 0)))).toBeTruthy(); + expect(ok(fx, edit::trim_start(clip_at(fx, 10), f(13), false))).toBeTruthy(); + expect(layout(fx)).toEqual("13+7@53 20+10@100"); + expect(ok(fx, edit::trim_start(clip_at(fx, 13), f(4), false))).toBeTruthy(); + expect(layout(fx)).toEqual("4+16@44 20+10@100"); + }); + + it("ripple trims the start, pulling the following clips", { + auto fx = three_clips(); + expect(ok(fx, edit::trim_start(clip_at(fx, 10), f(13), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+7@53 17+10@100"); + }); + + it("ripple trims the end outward, pushing the following clips", { + auto fx = three_clips(); + expect(ok(fx, edit::trim_end(clip_at(fx, 10), f(25), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+15@50 25+10@100"); + }); + + it("refuses to trim into a neighbour without ripple", { + auto fx = three_clips(); + expect(ok(fx, edit::trim_end(clip_at(fx, 10), f(25), false))).toBeFalsy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + }); + + it("refuses to trim before the start of the media", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 10, 3, 10); + expect(ok(fx, edit::trim_start(id, f(6), false))).toBeFalsy(); + expect(ok(fx, edit::trim_start(id, f(7), false))).toBeTruthy(); + expect(layout(fx)).toEqual("7+13@0"); + }); + + it("rolls an edit point both ways", { + auto fx = three_clips(); + const ClipId left = clip_at(fx, 0); + expect(ok(fx, edit::roll(left, f(13)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+13@0 13+7@53 20+10@100"); + expect(ok(fx, edit::roll(left, f(8)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+8@0 8+12@48 20+10@100"); + }); + + it("refuses to roll without an adjacent clip", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 0, 10); + place(fx, fx.video, 12, 50, 10); + expect(ok(fx, edit::roll(id, f(11)))).toBeFalsy(); + }); + + it("slips the source under a fixed range", { + auto fx = three_clips(); + expect(ok(fx, edit::slip(clip_at(fx, 10), f(5)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+10@55 20+10@100"); + expect(ok(fx, edit::slip(clip_at(fx, 0), f(-1)))).toBeFalsy(); + }); + + it("slides a clip between its neighbours", { + auto fx = three_clips(); + expect(ok(fx, edit::slide(clip_at(fx, 10), f(3)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+13@0 13+10@50 23+7@103"); + expect(ok(fx, edit::slide(clip_at(fx, 13), f(-5)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+8@0 8+10@50 18+12@98"); + }); + + it("slides into a gap without touching the far clip", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 10); + const ClipId id = place(fx, fx.video, 10, 50, 10); + place(fx, fx.video, 25, 100, 10); + expect(ok(fx, edit::slide(id, f(4)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+14@0 14+10@50 25+10@100"); + expect(ok(fx, edit::slide(id, f(2)))).toBeFalsy(); + }); + }); + + describe("timeline edits: moving, speed and properties", { + it("moves a clip to a free spot on another track", { + auto fx = three_clips(); + const ClipId id = clip_at(fx, 10); + expect(ok(fx, edit::move_clip(id, fx.overlay, f(40)))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 20+10@100"); + expect(layout(fx.editor.timeline(), fx.overlay)).toEqual("40+10@50"); + }); + + it("refuses to move onto another clip or onto a track of another kind", { + auto fx = three_clips(); + const ClipId id = clip_at(fx, 10); + expect(ok(fx, edit::move_clip(id, fx.video, f(15)))).toBeFalsy(); + expect(ok(fx, edit::move_clip(id, fx.audio, f(40)))).toBeFalsy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + }); + + it("doubles the speed, halving the duration and pulling later clips", { + auto fx = three_clips(); + expect(ok(fx, edit::set_speed(clip_at(fx, 10), q(2, 1), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+5@50 15+10@100"); + }); + + it("halves the speed, doubling the duration and pushing later clips", { + auto fx = three_clips(); + expect(ok(fx, edit::set_speed(clip_at(fx, 10), q(1, 2), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+20@50 30+10@100"); + // Trimming a slowed clip moves the source by half as much. + expect(ok(fx, edit::trim_start(clip_at(fx, 10), f(14), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+16@52 26+10@100"); + }); + + it("rounds an uneven speed down to the sequence grid", { + auto fx = three_clips(); + // 10 frames at 3x is 3.33 frames: 3 frames, staying inside the source range. + expect(ok(fx, edit::set_speed(clip_at(fx, 10), q(3, 1), true))).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+3@50 13+10@100"); + }); + + it("rejects a non-positive speed", { + auto fx = three_clips(); + expect(ok(fx, edit::set_speed(clip_at(fx, 10), q(0, 1), true))).toBeFalsy(); + }); + + it("changes video and audio properties, rejecting invalid ones", { + auto fx = three_clips(); + const ClipId id = clip_at(fx, 0); + VideoProperties v; + v.opacity = 0.5F; + v.transform.scale_x = 0.5; + expect(ok(fx, edit::set_video(id, v))).toBeTruthy(); + expect(fx.editor.timeline().find_clip(id)->video == v).toBeTruthy(); + v.opacity = 1.5F; + expect(ok(fx, edit::set_video(id, v))).toBeFalsy(); + AudioProperties a; + a.fade_in = f(4); + a.fade_out = f(6); + expect(ok(fx, edit::set_audio(id, a))).toBeTruthy(); + a.fade_out = f(7); + expect(ok(fx, edit::set_audio(id, a))).toBeFalsy(); + }); + }); + + describe("timeline edits: history", { + it("undoes and redoes every operation exactly", { + auto fx = three_clips(); + std::vector> commands; + commands.push_back(edit::split(clip_at(fx, 10), f(14))); + commands.push_back( + edit::insert(fx.video, fx.editor.new_clip_id(), f(5), camera(200, 4))); + commands.push_back(edit::roll(clip_at(fx, 0), f(3))); + commands.push_back(edit::trim_end(clip_at(fx, 20), f(30), true)); + commands.push_back(edit::ripple_delete_range(fx.video, f(2), f(7))); + commands.push_back(edit::set_speed(clip_at(fx, 0), q(1, 2), true)); + expect(undo_redo_round_trip(fx, std::move(commands))).toBeTruthy(); + }); + + it("records each compound operation as one history entry", { + auto fx = make_fixture(); + place(fx, fx.video, 0, 0, 30); + place(fx, fx.video, 10, 100, 5); + expect(std::string(fx.editor.undo_name())).toEqual("Overwrite"); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(layout(fx)).toEqual("0+30@0"); + }); + + it("groups commands in a transaction and rolls all back on failure", { + auto fx = three_clips(); + std::vector> good; + good.push_back(edit::remove_clip(clip_at(fx, 0))); + good.push_back(edit::remove_clip(clip_at(fx, 20))); + expect(ok(fx, edit::transaction("Delete Selection", std::move(good)))).toBeTruthy(); + expect(layout(fx)).toEqual("10+10@50"); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + + std::vector> bad; + bad.push_back(edit::remove_clip(clip_at(fx, 0))); + bad.push_back(edit::split(clip_at(fx, 10), f(10))); + expect(ok(fx, edit::transaction("Broken", std::move(bad)))).toBeFalsy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + }); + + it("clears the redo stack on a new edit", { + auto fx = three_clips(); + expect(ok(fx, edit::remove_clip(clip_at(fx, 0)))).toBeTruthy(); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(fx.editor.can_redo()).toBeTruthy(); + expect(ok(fx, edit::remove_clip(clip_at(fx, 20)))).toBeTruthy(); + expect(fx.editor.can_redo()).toBeFalsy(); + expect(fx.editor.redo().has_value()).toBeFalsy(); + }); + + it("restores a removed track with its clips", { + auto fx = three_clips(); + expect(ok(fx, edit::remove_track(fx.video))).toBeTruthy(); + expect(fx.editor.timeline().find_track(fx.video) == nullptr).toBeTruthy(); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + expect(fx.editor.timeline().tracks().front().id == fx.video).toBeTruthy(); + }); + + it("registers media outside the history", { + Editor ed(Timeline::create(oma::frame_rates::k30, q(1, 30)).value()); + const MediaInfo cam = camera_media(); + expect(ed.add_media(cam).has_value()).toBeTruthy(); + expect(ed.can_undo()).toBeFalsy(); + expect(ed.add_media(cam).has_value()).toBeFalsy(); + expect(ed.timeline().find_media(kCamera) != nullptr).toBeTruthy(); + }); + + it("forgets the history but keeps the timeline", { + auto fx = three_clips(); + expect(fx.editor.can_undo()).toBeTruthy(); + fx.editor.clear_history(); + expect(fx.editor.can_undo() || fx.editor.can_redo()).toBeFalsy(); + expect(layout(fx)).toEqual("0+10@0 10+10@50 20+10@100"); + }); + + it("never reuses an ID after undo", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 0, 30); + expect(ok(fx, edit::split(id, f(10)))).toBeTruthy(); + const ClipId first_tail = clip_at(fx, 10); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(ok(fx, edit::split(id, f(10)))).toBeTruthy(); + expect(clip_at(fx, 10) != first_tail).toBeTruthy(); + }); + + it("bumps the revision on execute, undo and redo", { + auto fx = three_clips(); + const auto r0 = fx.editor.revision(); + expect(ok(fx, edit::remove_clip(clip_at(fx, 0)))).toBeTruthy(); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(fx.editor.redo().has_value()).toBeTruthy(); + expect(fx.editor.revision() - r0).toBe(3U); + }); + + it("adds, sorts and removes markers through the history", { + auto fx = make_fixture(); + const MarkerId late = fx.editor.new_marker_id(); + const MarkerId early = fx.editor.new_marker_id(); + expect(ok(fx, edit::add_marker(late, f(90), "chorus"))).toBeTruthy(); + expect(ok(fx, edit::add_marker(early, f(30), "intro"))).toBeTruthy(); + expect(fx.editor.timeline().markers().front().name).toEqual("intro"); + expect(ok(fx, edit::remove_marker(early))).toBeTruthy(); + expect(fx.editor.timeline().markers().size()).toBe(1U); + expect(fx.editor.undo().has_value()).toBeTruthy(); + expect(fx.editor.timeline().markers().size()).toBe(2U); + const auto off_grid = oma::RationalTime::make(1, q(1, 60)).value(); + expect(ok(fx, edit::add_marker(fx.editor.new_marker_id(), off_grid, "x"))).toBeFalsy(); + }); + }); +} diff --git a/tests/timeline/test_evaluate.cpp b/tests/timeline/test_evaluate.cpp new file mode 100644 index 0000000..c0b0c8e --- /dev/null +++ b/tests/timeline/test_evaluate.cpp @@ -0,0 +1,128 @@ +#include "fixture.hpp" + +#include "oma/timeline/evaluate.hpp" + +#include "oma_test.hpp" + +using namespace timeline_test; + +namespace { + +Composition at(const Fixture& fx, std::int64_t frame) { + return evaluate(fx.editor.timeline(), f(frame)).value(); +} + +// Media time of the n-th video layer, in 30 fps frames of the 90 kHz camera timebase. +std::int64_t layer_frame(const Composition& c, std::size_t n) { + return c.video.at(n).media_time.value() / 3000; +} + +} // namespace + +void run_evaluate_tests() { + describe("timeline evaluation", { + it("is empty where nothing plays", { + auto fx = make_fixture(); + place(fx, fx.video, 10, 0, 10); + const Composition c = at(fx, 5); + expect(c.video.empty() && c.audio.empty()).toBeTruthy(); + }); + + it("maps timeline time to media time through the source in", { + auto fx = make_fixture(); + place(fx, fx.video, 10, 50, 20); + const Composition c = at(fx, 15); + expect(c.video.size()).toBe(1U); + expect(layer_frame(c, 0)).toBe(55LL); + expect(c.video[0].media_time.timebase() == q(1, 90000)).toBeTruthy(); + }); + + it("stacks video tracks bottom first and skips hidden tracks", { + auto fx = make_fixture(); + const ClipId base = place(fx, fx.video, 0, 0, 30); + const ClipId top = place(fx, fx.overlay, 0, 100, 30); + Composition c = at(fx, 3); + expect(c.video.size()).toBe(2U); + expect(c.video[0].clip == base && c.video[1].clip == top).toBeTruthy(); + expect(fx.editor.execute(edit::set_track_flags(fx.overlay, false, true)).has_value()) + .toBeTruthy(); + c = at(fx, 3); + expect(c.video.size()).toBe(1U); + expect(c.video[0].clip == base).toBeTruthy(); + }); + + it("applies the clip speed", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 40, 30); + expect(fx.editor.execute(edit::set_speed(id, q(2, 1), false)).has_value()).toBeTruthy(); + expect(layer_frame(at(fx, 10), 0)).toBe(60LL); + expect(at(fx, 10).audio.front().speed == q(2, 1)).toBeTruthy(); + }); + + it("carries the clip's visual properties", { + auto fx = make_fixture(); + const ClipId id = place(fx, fx.video, 0, 0, 30); + VideoProperties v; + v.opacity = 0.25F; + v.crop.left = 0.1; + expect(fx.editor.execute(edit::set_video(id, v)).has_value()).toBeTruthy(); + expect(at(fx, 0).video[0].video == v).toBeTruthy(); + }); + + it("plays the audio of video clips and audio tracks unless muted", { + auto fx = make_fixture(); + const ClipId cam = place(fx, fx.video, 0, 0, 30); + const ClipId music = fx.editor.new_clip_id(); + expect(fx.editor + .execute(edit::overwrite( + fx.audio, music, f(0), + edit::ClipSource{.media = kMusic, + .source_in = + oma::RationalTime::make(0, q(1, 48000)).value(), + .duration = f(30), + .time_map = {}, + .video = {}, + .audio = {}})) + .has_value()) + .toBeTruthy(); + Composition c = at(fx, 15); + expect(c.audio.size()).toBe(2U); + const AudioSource& m = c.audio[1]; + expect(m.clip == music).toBeTruthy(); + expect(m.media_time.value()).toBe(24000LL); // 15 frames at 48 kHz + expect(m.clip_offset == f(15) && m.clip_duration == f(30)).toBeTruthy(); + + AudioProperties muted; + muted.muted = true; + expect(fx.editor.execute(edit::set_audio(cam, muted)).has_value()).toBeTruthy(); + expect(fx.editor.execute(edit::set_track_flags(fx.audio, true, false)).has_value()) + .toBeTruthy(); + c = at(fx, 15); + expect(c.audio.empty()).toBeTruthy(); + expect(c.video.size()).toBe(1U); + }); + + it("shows a still image at any time", { + auto fx = make_fixture(); + const ClipId id = fx.editor.new_clip_id(); + expect(fx.editor + .execute(edit::overwrite(fx.overlay, id, f(0), + edit::ClipSource{.media = kPicture, + .source_in = {}, + .duration = f(900), + .time_map = {}, + .video = {}, + .audio = {}})) + .has_value()) + .toBeTruthy(); + const Composition c = at(fx, 899); + expect(c.video.size() == 1U && c.audio.empty()).toBeTruthy(); + }); + + it("rejects a time off the sequence grid", { + auto fx = make_fixture(); + const auto half = oma::RationalTime::make(1, q(1, 60)).value(); + expect(evaluate(fx.editor.timeline(), half).has_value()).toBeFalsy(); + }); + }); +} diff --git a/tests/timeline/test_model.cpp b/tests/timeline/test_model.cpp new file mode 100644 index 0000000..36cf7f0 --- /dev/null +++ b/tests/timeline/test_model.cpp @@ -0,0 +1,160 @@ +#include "fixture.hpp" + +#include "oma/timeline/evaluate.hpp" + +#include "oma_test.hpp" + +using namespace timeline_test; + +namespace { + +std::int64_t default_den(oma::FrameRate rate, int hz) { + return Timeline::default_timebase(rate, oma::SampleRate::make(hz).value()).value().den(); +} + +// A 23.976 fps sequence on its default timebase with a media file in a 1/15360 timebase (a +// common MP4 video timescale that is not a multiple of the frame grid). +struct OddGrid { + Editor editor; + TrackId track; + ClipId clip; +}; + +OddGrid odd_grid() { + const auto rate = oma::frame_rates::k23_976; + OddGrid g{Editor(Timeline::create(rate, oma::SampleRate::make(48000).value()).value()), {}, {}}; + const auto tb = q(1, 15360); + (void)g.editor.add_media(MediaInfo{.id = kCamera, + .start = oma::RationalTime::make(0, tb).value(), + .duration = oma::RationalTime::make(15360 * 60, tb).value(), + .has_video = true, + .has_audio = true, + .still = false}); + g.track = g.editor.new_track_id(); + (void)g.editor.execute(edit::add_track(g.track, TrackKind::Video, "V1")); + g.clip = g.editor.new_clip_id(); + const Timeline& tl = g.editor.timeline(); + const auto frame = rate.frame_to_time(1); // 1001/24000 s + const auto ten_frames = tl.at(tl.to_ticks(frame).value() * 10); + (void)g.editor.execute( + edit::overwrite(g.track, g.clip, tl.at(0), + edit::ClipSource{.media = kCamera, + .source_in = oma::RationalTime::make(15360, tb).value(), + .duration = ten_frames, + .time_map = {}, + .video = {}, + .audio = {}})); + return g; +} + +// Trims the start forward by `ticks` and back again; the source in must return exactly. +bool trim_round_trip_is_exact(std::int64_t ticks) { + OddGrid g = odd_grid(); + const Timeline& tl = g.editor.timeline(); + const auto original = tl.find_clip(g.clip)->source_in; + if (!g.editor.execute(edit::trim_start(g.clip, tl.at(ticks), false))) { + return false; + } + const bool moved = tl.find_clip(g.clip)->source_in != original; + if (!g.editor.execute(edit::trim_start(g.clip, tl.at(0), false))) { + return false; + } + return moved && tl.find_clip(g.clip)->source_in == original; +} + +} // namespace + +void run_model_tests() { + describe("Timeline model", { + it("picks a timebase that holds frames and samples", { + expect(default_den(oma::frame_rates::k29_97, 48000)).toBe(240000LL); + expect(default_den(oma::frame_rates::k23_976, 44100)).toBe(3528000LL); + expect(default_den(oma::frame_rates::k59_94, 96000)).toBe(480000LL); + expect(default_den(oma::frame_rates::k25, 48000)).toBe(48000LL); + const auto tl = + Timeline::create(oma::frame_rates::k29_97, oma::SampleRate::make(48000).value()); + expect(tl->to_ticks(oma::frame_rates::k29_97.frame_to_time(1)).value()).toBe(8008LL); + expect(tl->to_ticks(oma::SampleRate::make(48000).value().sample_to_time(1)).value()) + .toBe(5LL); + }); + + it("rejects a timebase that does not hold whole frames", { + expect(Timeline::create(oma::frame_rates::k29_97, q(1, 30)).has_value()).toBeFalsy(); + expect(Timeline::create(oma::frame_rates::k30, q(1, 30)).has_value()).toBeTruthy(); + }); + + it("converts exact times from other timebases and rejects inexact ones", { + const auto tl = Timeline::create(oma::frame_rates::k30, q(1, 30)).value(); + const auto exact = oma::RationalTime::make(6000, q(1, 90000)).value(); + const auto inexact = oma::RationalTime::make(1, q(1, 48000)).value(); + expect(tl.to_ticks(exact).value()).toBe(2LL); + expect(tl.to_ticks(inexact).has_value()).toBeFalsy(); + }); + + it("finds clips by half-open ranges", { + auto fx = make_fixture(); + const ClipId a = place(fx, fx.video, 0, 0, 10); + const ClipId b = place(fx, fx.video, 10, 50, 10); + const Timeline& tl = fx.editor.timeline(); + expect(tl.clip_at(fx.video, 9)->id == a).toBeTruthy(); + expect(tl.clip_at(fx.video, 10)->id == b).toBeTruthy(); + expect(tl.clip_at(fx.video, 20) == nullptr).toBeTruthy(); + expect(tl.track_of(b)->id == fx.video).toBeTruthy(); + expect(tl.duration() == f(20)).toBeTruthy(); + expect(tl.validate().has_value()).toBeTruthy(); + }); + + it("rejects non-positive speeds", { + expect(TimeMap::constant(q(0, 1)).has_value()).toBeFalsy(); + expect(TimeMap::constant(q(-1, 1)).has_value()).toBeFalsy(); + expect(TimeMap::constant(q(1, 2)).has_value()).toBeTruthy(); + }); + + it("edits audio at sample positions between frames", { + Editor ed(Timeline::create(oma::frame_rates::k30, oma::SampleRate::make(48000).value()) + .value()); + (void)ed.add_media( + MediaInfo{.id = kMusic, + .start = oma::RationalTime::make(0, q(1, 48000)).value(), + .duration = oma::RationalTime::make(48000, q(1, 48000)).value(), + .has_video = false, + .has_audio = true, + .still = false}); + const TrackId a = ed.new_track_id(); + (void)ed.execute(edit::add_track(a, TrackKind::Audio, "A1")); + const ClipId id = ed.new_clip_id(); + const auto second = oma::RationalTime::make(1, q(1, 1)).value(); + const auto zero = oma::RationalTime::make(0, q(1, 48000)).value(); + expect(ed.execute(edit::overwrite(a, id, ed.timeline().at(0), + edit::ClipSource{.media = kMusic, + .source_in = zero, + .duration = second, + .time_map = {}, + .video = {}, + .audio = {}})) + .has_value()) + .toBeTruthy(); + // Sample 1001 is not on the 30 fps frame grid (1600 samples per frame). + const auto sample = oma::RationalTime::make(1001, q(1, 48000)).value(); + expect(ed.execute(edit::split(id, sample)).has_value()).toBeTruthy(); + const Clip* tail = ed.timeline().clip_at(a, ed.timeline().to_ticks(sample).value()); + expect(tail->source_in == sample).toBeTruthy(); + }); + + it("keeps source positions exact through trims on a non-matching media grid", { + expect(trim_round_trip_is_exact(1)).toBeTruthy(); + expect(trim_round_trip_is_exact(10010)).toBeTruthy(); + }); + + it("rounds media times down to the media grid only on evaluation", { + OddGrid g = odd_grid(); + const Timeline& tl = g.editor.timeline(); + // The sequence tick is 1/48000 s, 0.32 of a 1/15360 media tick: still 15360. + auto at_one = evaluate(tl, tl.at(1)); + expect(at_one->video.front().media_time.value()).toBe(15360LL); + // One frame (2002 ticks) is 640.64 media ticks: floor to 640. + auto at_frame = evaluate(tl, tl.at(2002)); + expect(at_frame->video.front().media_time.value()).toBe(16000LL); + }); + }); +} From e88bd3e6ba2a5bc82842f429f0b8805711511c09 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:38:09 -0300 Subject: [PATCH 009/464] feat(spikes): add the s4 qt shared-device diagnostic --- Makefile | 13 +- PKGBUILD | 2 + tools/spikes/s4_qt_shared_device.cpp | 256 +++++++++++++++++++++++++++ 3 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 tools/spikes/s4_qt_shared_device.cpp diff --git a/Makefile b/Makefile index b01aa5f..c429f83 100644 --- a/Makefile +++ b/Makefile @@ -163,6 +163,17 @@ $(BUILD_DIR)/spikes/%: tools/spikes/%.cpp $(MAKEFILE_LIST) spikes: $(SPIKE_BINS) +# S4 reuses the real compositor and Qt's versioned RHI headers (limited compatibility API). +# Keep Qt discovery in this recipe: core library/test builds do not require Qt. +$(BUILD_DIR)/spikes/s4_qt_shared_device: tools/spikes/s4_qt_shared_device.cpp $(LIB_compositor) $(LIB_media) $(LIB_gpu) $(LIB_base) $(MAKEFILE_LIST) + $(call say,SPIKE,$@) + @mkdir -p $(@D) + $(Q)$(CXX) $(CXXFLAGS_BASE) $(TEST_WARNINGS) $(INC_compositor) \ + $$(pkg-config --cflags Qt6Quick) \ + -isystem $$(pkg-config --variable=includedir Qt6Gui)/QtGui/$$(pkg-config --modversion Qt6Gui) \ + -isystem $$(pkg-config --variable=includedir Qt6Gui)/QtGui/$$(pkg-config --modversion Qt6Gui)/QtGui \ + $< -o $@ $(LINK_compositor) $$(pkg-config --libs Qt6Quick) $(LDFLAGS_BASE) + all: libs tests compdb libs: $(ALL_LIBS) @@ -226,7 +237,7 @@ help: @echo ' format apply clang-format | format-check check without changing' @echo ' tidy run clang-tidy on the libs' @echo ' fixtures generate test media in tests/fixtures/generated (needs ffmpeg)' - @echo ' spikes build the M1 spikes in tools/spikes (needs Vulkan + FFmpeg headers)' + @echo ' spikes build the M1 spikes in tools/spikes (Vulkan + FFmpeg + Qt Quick/RHI)' @echo ' deps install the dependencies declared in the PKGBUILD (uses sudo pacman)' @echo ' clean remove build/$$BUILD | distclean remove all of build/' @echo 'Variables: BUILD=debug|release|asan|tsan CXX=g++|clang++ WERROR=1|0 V=1 (verbose)' diff --git a/PKGBUILD b/PKGBUILD index e6d69a5..d725b91 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -43,6 +43,8 @@ _devdepends=( 'vulkan-tools' # vulkaninfo (S1) 'vulkan-validation-layers' # validate synchronization in spikes and libs/gpu 'libva-utils' # vainfo (S1) + 'qt6-base' # Qt GUI and versioned RHI development headers (S4) + 'qt6-declarative' # Qt Quick (S4; move to depends when an app ships) ) source=("${pkgname}::git+${url}.git") diff --git a/tools/spikes/s4_qt_shared_device.cpp b/tools/spikes/s4_qt_shared_device.cpp new file mode 100644 index 0000000..6e33476 --- /dev/null +++ b/tools/spikes/s4_qt_shared_device.cpp @@ -0,0 +1,256 @@ +// S4: offscreen Qt Quick sampling the compositor's RGBA16F image on OmaMovie's device. +// This diagnostic deliberately runs all queue users sequentially. It is not a UI render loop +// or a display transform. See Docs/spikes/S4-qt-shared-device.md for scope and reproduction. +#include "oma/compositor/compositor.hpp" +#include "oma/gpu/device.hpp" +#include "oma/gpu/resources.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +std::atomic validation_errors = 0; + +VKAPI_ATTR VkBool32 VKAPI_CALL validation_message(VkDebugUtilsMessageSeverityFlagBitsEXT severity, + VkDebugUtilsMessageTypeFlagsEXT, + const VkDebugUtilsMessengerCallbackDataEXT* data, + void*) { + if (severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) { + ++validation_errors; + } + std::fprintf(stderr, "validation: %s\n", data->pMessage); + return VK_FALSE; +} + +class ImageItem final : public QQuickItem { +public: + explicit ImageItem(QQuickItem* parent, std::unique_ptr texture) + : QQuickItem(parent), texture_(std::move(texture)) { + setFlag(ItemHasContents); + } + +protected: + QSGNode* updatePaintNode(QSGNode* old, UpdatePaintNodeData*) override { + auto* node = static_cast(old); + if (!node) { + node = new QSGSimpleTextureNode; + auto* texture = window()->createTextureFromRhiTexture(texture_.get()); + if (!texture) { + delete node; + return nullptr; + } + node->setTexture(texture); + node->setOwnsTexture(true); // owns QRhi wrapper, NOT the compositor's VkImage + texture_.release(); + } + node->setRect(boundingRect()); + return node; + } + +private: + std::unique_ptr texture_; +}; + +bool check_pixels(QRhi& rhi, QRhiTexture& target, const std::array& expected) { + QRhiReadbackResult pixels; + bool completed = false; + pixels.completed = [&] { + completed = true; + }; + QRhiCommandBuffer* commands = nullptr; + if (rhi.beginOffscreenFrame(&commands) != QRhi::FrameOpSuccess) { + return false; + } + auto* updates = rhi.nextResourceUpdateBatch(); + updates->readBackTexture(QRhiReadbackDescription(&target), &pixels); + commands->resourceUpdate(updates); + if (rhi.endOffscreenFrame() != QRhi::FrameOpSuccess || rhi.finish() != QRhi::FrameOpSuccess || + !completed || pixels.format != QRhiTexture::RGBA8 || + pixels.pixelSize != target.pixelSize() || + pixels.data.size() != pixels.pixelSize.width() * pixels.pixelSize.height() * 4) { + return false; + } + // Diagnostic readback only: verify every pixel, outside the image-sharing path. + for (qsizetype i = 0; i < pixels.data.size(); ++i) { + const int actual = static_cast(pixels.data[i]); + const int wanted = + static_cast(std::lround(expected[static_cast(i % 4)] * 255)); + if (std::abs(actual - wanted) > 2) { + std::fprintf(stderr, "pixel mismatch at byte %lld: %d != %d\n", + static_cast(i), actual, wanted); + return false; + } + } + return true; +} + +bool run_scene(const oma::gpu::Device& device, QVulkanInstance& instance, + oma::compositor::VulkanCompositor& compositor, QSize size) { + // Qt cannot retrieve flagged queues. Fail before executing an invalid Vulkan call. + if (device.queue_create_flags() != 0) { + std::fprintf(stderr, "Qt 6.11 requires queues created with zero flags\n"); + return false; + } + QQuickRenderControl control; + QQuickWindow window(&control); + window.setVulkanInstance(&instance); + window.setGraphicsDevice(QQuickGraphicsDevice::fromDeviceObjects( + device.physical_device(), device.device(), device.graphics_family(), 0)); + window.setGeometry(0, 0, size.width(), size.height()); + if (!control.initialize()) { + std::fprintf(stderr, "QQuickRenderControl initialization failed\n"); + return false; + } + auto* rhi = control.rhi(); + const auto* handles = static_cast(rhi->nativeHandles()); + if (rhi->backend() != QRhi::Vulkan || !handles || handles->dev != device.device() || + handles->physDev != device.physical_device() || + handles->gfxQueue != device.queue(device.graphics_family(), 0)) { + std::fprintf(stderr, "Qt native handles differ from OmaMovie handles\n"); + return false; + } + std::printf("Qt %s: same physical device, device and queue; %dx%d\n", qVersion(), size.width(), + size.height()); + + oma::compositor::RenderGraph graph; + graph.width = static_cast(size.width()); + graph.height = static_cast(size.height()); + if (!compositor.render(graph, {})) { + return false; + } + auto imported = std::unique_ptr(rhi->newTexture(QRhiTexture::RGBA16F, size)); + const auto native_image = reinterpret_cast(compositor.output()->handle()); + if (!imported->createFrom({native_image, VK_IMAGE_LAYOUT_GENERAL})) { + return false; + } + auto target = std::unique_ptr( + rhi->newTexture(QRhiTexture::RGBA8, size, 1, + QRhiTexture::RenderTarget | QRhiTexture::UsedAsTransferSource)); + if (!target->create()) { + return false; + } + auto render_target = std::unique_ptr( + rhi->newTextureRenderTarget(QRhiTextureRenderTargetDescription(target.get()))); + auto pass = std::unique_ptr( + render_target->newCompatibleRenderPassDescriptor()); + render_target->setRenderPassDescriptor(pass.get()); + if (!render_target->create()) { + return false; + } + window.setRenderTarget(QQuickRenderTarget::fromRhiRenderTarget(render_target.get())); + // Stack item outlives rendering; the node owns the transferred QRhi wrapper. + auto* shared_texture = imported.get(); + ImageItem item(window.contentItem(), std::move(imported)); + item.setSize(size); + bool ok = true; + for (int frame = 0; frame < 30 && ok; ++frame) { + graph.background = frame % 2 == 0 ? std::array{0.25F, 0.5F, 0.75F, 1.0F} + : std::array{0.75F, 0.25F, 0.5F, 1.0F}; + // ponytail: synchronous single-thread spike, no workers or window presentation. + // A production bridge needs a shared submission protocol and frame lifetime tracking. + ok = compositor.render(graph, {}).has_value(); + if (!ok) { + break; + } + shared_texture->setNativeLayout(VK_IMAGE_LAYOUT_GENERAL); + item.update(); + control.polishItems(); + control.beginFrame(); + control.sync(); + control.render(); + control.endFrame(); + // No next compositor write/destruction until all Qt reads have completed. + ok = rhi->finish() == QRhi::FrameOpSuccess; + if (ok && (frame == 0 || frame == 29)) { + ok = check_pixels(*rhi, *target, graph.background); + } + } + // Invalidate nodes/wrappers BEFORE resizing or destroying the compositor image. + window.setRenderTarget({}); + render_target.reset(); + pass.reset(); + target.reset(); + control.invalidate(); + std::printf("30 alternating frames, diagnostic pixel checks: %s\n", ok ? "PASS" : "FAIL"); + return ok; +} + +} // namespace + +int main(int argc, char** argv) { + QGuiApplication app(argc, argv); + QQuickWindow::setGraphicsApi(QSGRendererInterface::Vulkan); + oma::gpu::DeviceOptions options; + options.internally_synchronized_queues = false; + options.instance_extensions = {VK_KHR_SURFACE_EXTENSION_NAME, "VK_KHR_wayland_surface", + VK_EXT_DEBUG_UTILS_EXTENSION_NAME}; + options.instance_layers = {"VK_LAYER_KHRONOS_validation"}; + auto created = oma::gpu::Device::create(options); + if (!created) { + std::fprintf(stderr, "%s\n", created.error().summary().c_str()); + return 1; + } + auto device = std::move(*created); + std::printf("%s / %s; queue flags: %u; validation enabled\n", device->info().name.c_str(), + device->info().driver_info.c_str(), device->queue_create_flags()); + const auto create_debug = reinterpret_cast( + device->instance_proc_addr()(device->instance(), "vkCreateDebugUtilsMessengerEXT")); + const auto destroy_debug = reinterpret_cast( + device->instance_proc_addr()(device->instance(), "vkDestroyDebugUtilsMessengerEXT")); + VkDebugUtilsMessengerEXT messenger = VK_NULL_HANDLE; + const VkDebugUtilsMessengerCreateInfoEXT debug_info{ + .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT, + .pNext = nullptr, + .flags = 0, + .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT, + .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT, + .pfnUserCallback = validation_message, + .pUserData = nullptr}; + if (!create_debug || !destroy_debug || + create_debug(device->instance(), &debug_info, nullptr, &messenger) != VK_SUCCESS) { + return 1; + } + bool ok = false; + { + QVulkanInstance instance; + instance.setVkInstance(device->instance()); // borrowed: Device remains the owner + instance.setApiVersion(QVersionNumber(1, 4)); + auto compositor = oma::compositor::VulkanCompositor::create(*device); + const bool instance_ok = instance.create(); + if (!instance_ok) { + std::fprintf(stderr, "Qt instance wrapper failed: %d\n", instance.errorCode()); + } + if (!compositor) { + std::fprintf(stderr, "%s\n", compositor.error().summary().c_str()); + } + ok = instance_ok && compositor && + run_scene(*device, instance, **compositor, QSize(320, 180)) && + run_scene(*device, instance, **compositor, QSize(640, 360)); + } // Qt, then compositor resources; VkDevice and validation callback still alive + const VkInstance native_instance = device->instance(); + // Device owns the instance, so the messenger must be destroyed before Device. + destroy_debug(native_instance, messenger, nullptr); + device.reset(); + std::printf("validation errors: %u; S4 offscreen interop: %s\n", validation_errors.load(), + ok && validation_errors == 0 ? "PASS" : "FAIL"); + return ok && validation_errors == 0 ? 0 : 1; +} From 83e748ade1f91649f135196d85c7900d999f1983 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:38:09 -0300 Subject: [PATCH 010/464] feat(ui): add the qt quick editor shell with timeline editing and audio-clock playback --- CLAUDE.md | 1 + Makefile | 29 +- apps/omamovie/qml/Field.qml | 10 + apps/omamovie/qml/Icon.qml | 62 ++ apps/omamovie/qml/Main.qml | 957 ++++++++++++++++++++++++++ apps/omamovie/qml/OmaAction.qml | 8 + apps/omamovie/qml/OmaButton.qml | 61 ++ apps/omamovie/qml/Separator.qml | 3 + apps/omamovie/src/audio_player.cpp | 170 +++++ apps/omamovie/src/audio_player.hpp | 89 +++ apps/omamovie/src/frame_source.cpp | 112 +++ apps/omamovie/src/frame_source.hpp | 59 ++ apps/omamovie/src/main.cpp | 495 +++++++++++++ apps/omamovie/src/preview_item.cpp | 140 ++++ apps/omamovie/src/preview_item.hpp | 77 +++ apps/omamovie/src/session.cpp | 669 ++++++++++++++++++ apps/omamovie/src/session.hpp | 237 +++++++ apps/omamovie/src/timeline_audio.cpp | 164 +++++ apps/omamovie/src/timeline_audio.hpp | 55 ++ apps/omamovie/src/video_scheduler.cpp | 78 +++ apps/omamovie/src/video_scheduler.hpp | 72 ++ apps/omamovie/src/viewer_frame.cpp | 55 ++ apps/omamovie/src/viewer_frame.hpp | 32 + 23 files changed, 3634 insertions(+), 1 deletion(-) create mode 100644 apps/omamovie/qml/Field.qml create mode 100644 apps/omamovie/qml/Icon.qml create mode 100644 apps/omamovie/qml/Main.qml create mode 100644 apps/omamovie/qml/OmaAction.qml create mode 100644 apps/omamovie/qml/OmaButton.qml create mode 100644 apps/omamovie/qml/Separator.qml create mode 100644 apps/omamovie/src/audio_player.cpp create mode 100644 apps/omamovie/src/audio_player.hpp create mode 100644 apps/omamovie/src/frame_source.cpp create mode 100644 apps/omamovie/src/frame_source.hpp create mode 100644 apps/omamovie/src/main.cpp create mode 100644 apps/omamovie/src/preview_item.cpp create mode 100644 apps/omamovie/src/preview_item.hpp create mode 100644 apps/omamovie/src/session.cpp create mode 100644 apps/omamovie/src/session.hpp create mode 100644 apps/omamovie/src/timeline_audio.cpp create mode 100644 apps/omamovie/src/timeline_audio.hpp create mode 100644 apps/omamovie/src/video_scheduler.cpp create mode 100644 apps/omamovie/src/video_scheduler.hpp create mode 100644 apps/omamovie/src/viewer_frame.cpp create mode 100644 apps/omamovie/src/viewer_frame.hpp diff --git a/CLAUDE.md b/CLAUDE.md index 76257df..97a1115 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,7 @@ make format | make format-check make tidy # clang-tidy on the libs (warnings are errors) make fixtures # regenerate test media in tests/fixtures/generated/ (ffmpeg) make spikes # M1 spikes in tools/spikes (Docs/spikes/) +make run-gui # Qt/Vulkan editor shell; GUI_FILE=path or RUN_GUI_SMOKE=1 makepkg -si # build and install the Arch package from the PKGBUILD ``` diff --git a/Makefile b/Makefile index c429f83..b4852a7 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,7 @@ $(BUILD_DIR)/obj/%.o: %.cpp $(MAKEFILE_LIST) -include $(patsubst %.cpp,$(BUILD_DIR)/obj/%.d,$(ALL_SOURCES)) -.PHONY: all libs tests test clean distclean compdb format format-check tidy fixtures spikes deps help +.PHONY: all libs tests test clean distclean compdb format format-check tidy fixtures spikes run-gui deps help # Spikes (tools/spikes/*.cpp): disposable single-file experiments (Docs/spikes/), built only on # request because they need the Vulkan and FFmpeg development files. pkg-config runs inside the @@ -174,6 +174,32 @@ $(BUILD_DIR)/spikes/s4_qt_shared_device: tools/spikes/s4_qt_shared_device.cpp $( -isystem $$(pkg-config --variable=includedir Qt6Gui)/QtGui/$$(pkg-config --modversion Qt6Gui)/QtGui \ $< -o $@ $(LINK_compositor) $$(pkg-config --libs Qt6Quick) $(LDFLAGS_BASE) +# The app: every source in apps/omamovie/src plus moc output for headers declaring Q_OBJECT. +# Built in one compiler call; QML is loaded from the source tree at run time. +APP_SOURCES := $(wildcard apps/omamovie/src/*.cpp) +APP_HEADERS := $(wildcard apps/omamovie/src/*.hpp) +APP_MOC_HEADERS := $(shell grep -l Q_OBJECT $(APP_HEADERS) 2>/dev/null) +APP_MOCS := $(patsubst apps/omamovie/src/%.hpp,$(BUILD_DIR)/gen/omamovie/moc_%.cpp,$(APP_MOC_HEADERS)) + +$(BUILD_DIR)/gen/omamovie/moc_%.cpp: apps/omamovie/src/%.hpp $(MAKEFILE_LIST) + $(call say,MOC,$<) + @mkdir -p $(@D) + $(Q)/usr/lib/qt6/moc $< -o $@ + +# The viewer wraps compositor images with Qt's RHI (limited compatibility API, versioned headers). +$(BUILD_DIR)/omamovie: $(APP_SOURCES) $(APP_HEADERS) $(APP_MOCS) $(wildcard apps/omamovie/qml/*.qml) $(LIB_timeline) $(LIB_audio) $(LIB_compositor) $(LIB_media) $(LIB_gpu) $(LIB_base) $(MAKEFILE_LIST) + $(call say,GUI,$@) + @mkdir -p $(@D) + $(Q)$(CXX) $(CXXFLAGS_BASE) -fPIC $(TEST_WARNINGS) $(INC_timeline) $(INC_audio) $(INC_compositor) -Iapps/omamovie/src \ + $$(pkg-config --cflags Qt6Quick Qt6Test) \ + -isystem $$(pkg-config --variable=includedir Qt6Gui)/QtGui/$$(pkg-config --modversion Qt6Gui)/QtGui \ + $(APP_SOURCES) $(APP_MOCS) -o $@ $(LINK_timeline) $(LINK_audio) $(LINK_compositor) \ + $$(pkg-config --libs Qt6Quick Qt6Test) $(LDFLAGS_BASE) + +run-gui: $(BUILD_DIR)/omamovie + $(if $(filter 1,$(RUN_GUI_SMOKE)),@test -f tests/fixtures/generated/hevc_10bit.mp4 || $(MAKE) fixtures) + $(Q)$(BUILD_DIR)/omamovie $(if $(filter 1,$(RUN_GUI_SMOKE)),--smoke,$(GUI_FILE)) + all: libs tests compdb libs: $(ALL_LIBS) @@ -238,6 +264,7 @@ help: @echo ' tidy run clang-tidy on the libs' @echo ' fixtures generate test media in tests/fixtures/generated (needs ffmpeg)' @echo ' spikes build the M1 spikes in tools/spikes (Vulkan + FFmpeg + Qt Quick/RHI)' + @echo ' run-gui open the Qt editor shell [GUI_FILE=path; RUN_GUI_SMOKE=1]' @echo ' deps install the dependencies declared in the PKGBUILD (uses sudo pacman)' @echo ' clean remove build/$$BUILD | distclean remove all of build/' @echo 'Variables: BUILD=debug|release|asan|tsan CXX=g++|clang++ WERROR=1|0 V=1 (verbose)' diff --git a/apps/omamovie/qml/Field.qml b/apps/omamovie/qml/Field.qml new file mode 100644 index 0000000..7f18c8b --- /dev/null +++ b/apps/omamovie/qml/Field.qml @@ -0,0 +1,10 @@ +import QtQuick +import QtQuick.Layouts + +ColumnLayout { + property string label + property string value + spacing: 1 + Text { text: parent.label; color: colors.dark_foreground; font.pixelSize: 9; font.bold: true } + Text { text: parent.value; color: colors.foreground; font.pixelSize: 11; elide: Text.ElideMiddle; Layout.maximumWidth: 360 } +} diff --git a/apps/omamovie/qml/Icon.qml b/apps/omamovie/qml/Icon.qml new file mode 100644 index 0000000..07d2f50 --- /dev/null +++ b/apps/omamovie/qml/Icon.qml @@ -0,0 +1,62 @@ +import QtQuick +import QtQuick.Shapes + +// Our own monochrome set on a 24 px grid, 1.5 px stroke, tinted at runtime (ui-design §10.2). +// Drawn as SVG paths with Qt Quick Shapes until SVG files can ship through qt6-svg. +Item { + id: icon + property string name + property color color: colors.foreground + property real size: 18 + implicitWidth: size + implicitHeight: size + readonly property var paths: ({ + "back": "M15 6l-6 6 6 6", + "undo": "M9 14L4 9l5-5M4 9h10a5.5 5.5 0 0 1 0 11h-3", + "redo": "M15 14l5-5-5-5M20 9H10a5.5 5.5 0 0 0 0 11h3", + "import": "M12 4v11M7.5 10.5L12 15l4.5-4.5M5 20h14", + "export": "M12 15V4M7.5 8.5L12 4l4.5 4.5M5 20h14", + "library": "M4 5h6v14H4zM14 5h6v6h-6zM14 15h6v4h-6z", + "color": "M12 4a8 8 0 1 0 0 16a8 8 0 1 0 0-16zM12 4v16", + "crop": "M7 3v14h14M3 7h14v14", + "volume": "M4 9.5h3.5L12 6v12l-4.5-3.5H4zM15.5 9.5a3.5 3.5 0 0 1 0 5M18 7a7 7 0 0 1 0 10", + "speed": "M12 20a8 8 0 1 1 0-16a8 8 0 1 1 0 16zM12 12l3.5-3.5M12 4v2.5", + "effects": "M12 4l1.8 6.2L20 12l-6.2 1.8L12 20l-1.8-6.2L4 12l6.2-1.8z", + "overlay": "M4 4h11v11H4zM9 9h11v11H9z", + "info": "M12 4a8 8 0 1 0 0 16a8 8 0 1 0 0-16zM12 11v5M12 8v.6", + "play": "M8 5.5v13l10-6.5z", + "pause": "M9 5.5v13M15 5.5v13", + "stop": "M6 6h12v12H6z", + "start": "M6 5v14M17 6l-7 6 7 6z", + "end": "M18 5v14M7 6l7 6-7 6z", + "backward": "M12 6l-7 6 7 6zM19 6l-7 6 7 6z", + "forward": "M5 6l7 6-7 6zM12 6l7 6-7 6z", + "previous": "M17 6l-7 6 7 6zM7 6v12", + "next": "M7 6l7 6-7 6zM17 6v12", + "append": "M4 7h9v10H4zM17 8v8M13 12h8", + "insert": "M4 7h5v10H4zM15 7h5v10h-5zM12 5v14", + "overwrite": "M4 7h16v10H4zM10 10h4v4h-4z", + "split": "M12 4v16M4 8h5v8H4zM15 8h5v8h-5z", + "rippleDelete": "M4 8h5v8H4zM15 8h5v8h-5zM10 10l4 4M14 10l-4 4", + "lift": "M4 8h16v9H4zM12 12V3M9 6l3-3 3 3", + "fullscreen": "M4 9V4h5M15 4h5v5M20 15v5h-5M9 20H4v-5", + "zoomIn": "M10.5 4a6.5 6.5 0 1 0 0 13a6.5 6.5 0 1 0 0-13zM15.5 15.5L20 20M7.5 10.5h6M10.5 7.5v6", + "zoomOut": "M10.5 4a6.5 6.5 0 1 0 0 13a6.5 6.5 0 1 0 0-13zM15.5 15.5L20 20M7.5 10.5h6", + "fit": "M4 12h16M7 9l-3 3 3 3M17 9l3 3-3 3" + }) + Shape { + width: 24 + height: 24 + scale: icon.size / 24 + transformOrigin: Item.TopLeft + preferredRendererType: Shape.CurveRenderer + ShapePath { + strokeColor: icon.color + strokeWidth: 1.5 + fillColor: "transparent" + capStyle: ShapePath.RoundCap + joinStyle: ShapePath.RoundJoin + PathSvg { path: icon.paths[icon.name] || "" } + } + } +} diff --git a/apps/omamovie/qml/Main.qml b/apps/omamovie/qml/Main.qml new file mode 100644 index 0000000..a697c07 --- /dev/null +++ b/apps/omamovie/qml/Main.qml @@ -0,0 +1,957 @@ +import QtQuick +import QtQuick.Controls +import QtQuick.Dialogs +import QtQuick.Layouts +import QtQuick.Shapes +import OmaMovie 1.0 + +// Editor shell following Docs/ui-design.md: three fixed areas (library, viewer, timeline), +// adjustments in a bar and drawer above the viewer, a transport, and every command +// registered once as an Action with its §8.1 shortcut. +ApplicationWindow { + id: root + visible: false + width: 1280 + height: 800 + minimumWidth: 640 + minimumHeight: 520 + title: session.editing ? "Untitled project — OmaMovie" : "OmaMovie — Projects" + color: colors.background + + property color fg: colors.foreground + property color muted: colors.dark_foreground + property color accent: colors.accent + property color line: colors.lighter_background + readonly property color viewerBackground: "#1b1b1d" // never themed (ui-design §5) + // Layout breakpoints (ui-design §2): wide ≥ 1500, half width 900–1500, narrow below. + property bool compact: width < 1500 + property bool narrow: width < 900 + property bool libraryOverlay: false + property bool viewerOnly: false + property string drawer: "" // the open adjustment, "" when the drawer is closed + property real upperRatio: 0.55 + property real pixelsPerSecond: 60 + property bool fitTimeline: true + + function pad(n, width) { return String(n).padStart(width, "0") } + // HH:MM:SS:FF at the nominal frame rate (display only). + function timecode(seconds) { + const fps = Math.max(1, Math.round(session.frameRate)) + const total = Math.floor(Math.max(0, seconds) * fps + 1e-6) + const s = Math.floor(total / fps) + return pad(Math.floor(s / 3600), 2) + ":" + pad(Math.floor(s / 60) % 60, 2) + ":" + + pad(s % 60, 2) + ":" + pad(total % fps, 2) + } + readonly property bool audioAdjusted: !!session.info.clip && + (session.info.gain !== 1 || session.info.fadeIn > 0 || session.info.fadeOut > 0 || session.info.muted) + function decibels(gain) { + return gain <= 0.0001 ? "−∞ dB" : (20 * Math.log(gain) / Math.LN10).toFixed(1) + " dB" + } + function shortcutText(action) { + return action && action.keys ? " (" + action.keys + ")" : "" + } + + // ---------------------------------------------------------------- actions (ui-design §8.1) + // The single registry of editor commands: name, default shortcut and handler. Buttons and + // shortcuts both go through it, so remapping later touches one place. + QtObject { + id: actions + readonly property bool editing: session.editing + readonly property bool media: session.hasMedia + + property OmaAction playPause: OmaAction { + text: session.playing ? "Pause" : "Play" + keys: "Space"; enabled: actions.media + onTriggered: session.togglePlay() + } + property OmaAction pause: OmaAction { + text: "Pause"; keys: "K"; enabled: actions.media && session.playing + onTriggered: session.pause() + } + property OmaAction stop: OmaAction { + text: "Stop and return to start"; enabled: actions.media + onTriggered: { session.pause(); session.seek(0) } + } + property OmaAction playForward: OmaAction { + text: "Play forward (repeat: faster)"; keys: "L"; enabled: actions.media + onTriggered: session.shuttle(1) + } + property OmaAction playBackward: OmaAction { + text: "Play backward (repeat: faster)"; keys: "J"; enabled: actions.media + onTriggered: session.shuttle(-1) + } + property OmaAction previousFrame: OmaAction { + text: "Previous frame"; keys: "Left"; enabled: actions.media + onTriggered: session.stepFrames(-1) + } + property OmaAction nextFrame: OmaAction { + text: "Next frame"; keys: "Right"; enabled: actions.media + onTriggered: session.stepFrames(1) + } + property OmaAction back10: OmaAction { + text: "Back 10 frames"; keys: "Shift+Left"; enabled: actions.media + onTriggered: session.stepFrames(-10) + } + property OmaAction forward10: OmaAction { + text: "Forward 10 frames"; keys: "Shift+Right"; enabled: actions.media + onTriggered: session.stepFrames(10) + } + property OmaAction toStart: OmaAction { + text: "Go to start"; keys: "Home"; enabled: actions.media + onTriggered: session.seek(0) + } + property OmaAction toEnd: OmaAction { + text: "Go to end"; keys: "End"; enabled: actions.media + onTriggered: session.toEnd() + } + readonly property bool mediaChosen: actions.editing && session.selectedMedia >= 0 + property OmaAction append: OmaAction { + text: "Append to the end"; keys: "E"; enabled: actions.mediaChosen + onTriggered: { session.appendSelected(); root.libraryOverlay = false } + } + property OmaAction insert: OmaAction { + text: "Insert at the playhead"; keys: "W"; enabled: actions.mediaChosen + onTriggered: { session.insertSelected(); root.libraryOverlay = false } + } + property OmaAction overwrite: OmaAction { + text: "Overwrite at the playhead"; keys: "D"; enabled: actions.mediaChosen + onTriggered: { session.overwriteSelected(); root.libraryOverlay = false } + } + property OmaAction split: OmaAction { + text: "Split at the playhead"; keys: "Ctrl+B"; enabled: actions.media + onTriggered: session.splitAtPlayhead() + } + property OmaAction remove: OmaAction { + text: "Delete and close the gap"; keys: "Delete"; enabled: actions.media + onTriggered: session.deleteSelected(true) + } + property OmaAction lift: OmaAction { + text: "Replace with a gap"; keys: "Shift+Delete"; enabled: actions.media + onTriggered: session.deleteSelected(false) + } + property OmaAction undo: OmaAction { + text: session.canUndo ? "Undo " + session.undoText : "Undo" + keys: "Ctrl+Z"; enabled: actions.editing && session.canUndo + onTriggered: session.undo() + } + property OmaAction redo: OmaAction { + text: session.canRedo ? "Redo " + session.redoText : "Redo" + keys: "Ctrl+Shift+Z"; enabled: actions.editing && session.canRedo + onTriggered: session.redo() + } + property OmaAction importMedia: OmaAction { + text: "Import"; keys: "Ctrl+I" + onTriggered: fileDialog.open() + } + property OmaAction exportMovie: OmaAction { text: "Export (M7)"; keys: "Ctrl+E"; enabled: false } + property OmaAction toggleLibrary: OmaAction { + text: "Library"; keys: "Ctrl+1"; enabled: actions.editing && root.compact + onTriggered: root.libraryOverlay = !root.libraryOverlay + } + property OmaAction fullViewer: OmaAction { + text: "Full-screen viewer"; keys: "Ctrl+Shift+F"; enabled: actions.editing + onTriggered: root.viewerOnly = !root.viewerOnly + } + property OmaAction leaveFullViewer: OmaAction { + text: "Leave full screen"; keys: "Escape"; enabled: root.viewerOnly + onTriggered: root.viewerOnly = false + } + property OmaAction zoomIn: OmaAction { + text: "Zoom in"; keys: "Ctrl+="; enabled: actions.editing + onTriggered: { root.fitTimeline = false; root.pixelsPerSecond = Math.min(2400, timelinePanel.scale * 1.5) } + } + property OmaAction zoomOut: OmaAction { + text: "Zoom out"; keys: "Ctrl+-"; enabled: actions.editing + onTriggered: { root.fitTimeline = false; root.pixelsPerSecond = Math.max(4, timelinePanel.scale / 1.5) } + } + property OmaAction zoomFit: OmaAction { + text: "Fit the project"; keys: "Shift+Z"; enabled: actions.editing + onTriggered: root.fitTimeline = true + } + property OmaAction volume: OmaAction { + text: "Volume"; keys: "Ctrl+Shift+V" + enabled: actions.editing && !!session.info.clip && !!session.info.hasAudio + onTriggered: root.drawer = root.drawer === "volume" ? "" : "volume" + } + property OmaAction info: OmaAction { + text: "Info"; keys: "Ctrl+Shift+I"; enabled: actions.editing && !!session.info.name; checkable: true + checked: root.drawer === "info" + onTriggered: root.drawer = root.drawer === "info" ? "" : "info" + } + } + + FileDialog { + id: fileDialog + title: "Import" + fileMode: FileDialog.OpenFiles + nameFilters: ["Videos and pictures (*.mp4 *.mkv *.mov *.webm *.avi *.m4v *.y4m *.png *.jpg *.jpeg)", "All files (*)"] + onAccepted: { + for (const file of selectedFiles) session.importUrl(file) + root.libraryOverlay = false + } + } + + // ---------------------------------------------------------------- Projects screen (§3) + Item { + anchors.fill: parent + visible: !session.editing + + RowLayout { + id: projectsBar + anchors.left: parent.left + anchors.right: parent.right + anchors.leftMargin: 24 + anchors.rightMargin: 24 + height: 60 + Text { text: "OmaMovie"; color: root.fg; font.pixelSize: 18; font.bold: true } + Item { Layout.fillWidth: true } + OmaButton { text: "New project"; primary: true; onClicked: session.newProject() } + } + Rectangle { anchors.top: projectsBar.bottom; width: parent.width; height: 1; color: root.line } + + ColumnLayout { + anchors.top: projectsBar.bottom + anchors.left: parent.left + anchors.right: parent.right + anchors.margins: 32 + spacing: 14 + Text { + Layout.topMargin: 16 + text: "RECENT" + color: root.muted + font.pixelSize: 11 + font.bold: true + font.letterSpacing: 1 + } + Rectangle { + Layout.preferredWidth: 280 + Layout.preferredHeight: 170 + radius: 8 + color: colors.dark_background + visible: session.media.length > 0 + ColumnLayout { + anchors.fill: parent + anchors.margins: 14 + spacing: 6 + Rectangle { + Layout.fillWidth: true + Layout.fillHeight: true + radius: 4 + color: root.viewerBackground + Image { + anchors.fill: parent + fillMode: Image.PreserveAspectCrop + source: session.media.length ? session.media[0].thumbnail : "" + } + } + Text { text: "Untitled project"; color: root.fg; font.pixelSize: 13; font.bold: true } + Text { + text: session.media.length + " video" + (session.media.length === 1 ? "" : "s") + " · this session" + color: root.muted + font.pixelSize: 11 + } + } + MouseArea { anchors.fill: parent; cursorShape: Qt.PointingHandCursor; onClicked: session.continueProject() } + } + Text { + visible: session.media.length === 0 + text: "No projects yet. Start one with New project." + color: root.muted + font.pixelSize: 12 + } + Text { + text: "Projects are kept for this session only; saving arrives with M7." + color: root.muted + font.pixelSize: 11 + } + } + } + + // ---------------------------------------------------------------- Edit screen + Item { + id: editor + anchors.fill: parent + visible: session.editing + + // Top bar (§2.1): back to Projects, project name, undo/redo, import, export. + Rectangle { + id: topbar + visible: !root.viewerOnly + anchors.left: parent.left + anchors.right: parent.right + height: visible ? 48 : 0 + color: colors.background + RowLayout { + anchors.fill: parent + anchors.leftMargin: 8 + anchors.rightMargin: 10 + spacing: 4 + OmaButton { + iconName: "back" + text: "Projects" + showLabel: !root.compact + tip: "Projects" + onClicked: session.showProjects() + } + Text { + Layout.leftMargin: 6 + Layout.fillWidth: true + text: "Untitled project" + color: root.fg + font.pixelSize: 13 + font.bold: true + elide: Text.ElideRight + } + OmaButton { action: actions.undo; iconName: "undo"; showLabel: false } + OmaButton { action: actions.redo; iconName: "redo"; showLabel: false } + Separator { Layout.leftMargin: 6; Layout.rightMargin: 6 } + OmaButton { action: actions.importMedia; iconName: "import"; showLabel: !root.compact } + OmaButton { action: actions.exportMovie; iconName: "export"; text: "Export"; showLabel: !root.compact } + } + Rectangle { anchors.bottom: parent.bottom; width: parent.width; height: 1; color: root.line } + } + + Item { + id: workspace + anchors.top: topbar.bottom + anchors.bottom: parent.bottom + anchors.left: parent.left + anchors.right: parent.right + + Item { + id: upper + width: parent.width + height: root.viewerOnly ? parent.height : parent.height * root.upperRatio + + // Library (§4): docked at ~30% in wide windows, an overlay panel otherwise. + Rectangle { + id: library + z: 3 + visible: !root.viewerOnly && (!root.compact || root.libraryOverlay) + width: root.compact ? Math.min(380, upper.width * 0.8) : upper.width * 0.30 + // As an overlay it stops above the transport so the timecode stays visible. + height: root.compact ? upper.height - transport.height : upper.height + color: colors.dark_background + + RowLayout { + anchors.fill: parent + spacing: 0 + ColumnLayout { + Layout.preferredWidth: 104 + Layout.fillHeight: true + Layout.margins: 8 + spacing: 4 + Text { text: "SOURCES"; color: root.muted; font.pixelSize: 10; font.bold: true; Layout.bottomMargin: 4 } + OmaButton { text: "Project"; selected: true; Layout.fillWidth: true } + Item { Layout.fillHeight: true } + } + Rectangle { Layout.fillHeight: true; implicitWidth: 1; color: root.line } + ColumnLayout { + Layout.fillWidth: true + Layout.fillHeight: true + spacing: 0 + RowLayout { + Layout.fillWidth: true + Layout.preferredHeight: 40 + Layout.leftMargin: 12 + Layout.rightMargin: 8 + Text { text: "Media"; color: root.fg; font.pixelSize: 12; font.bold: true } + Text { text: session.media.length; color: root.muted; font.pixelSize: 11 } + Item { Layout.fillWidth: true } + OmaButton { action: actions.importMedia; iconName: "import"; showLabel: false } + } + GridView { + id: mediaGrid + Layout.fillWidth: true + Layout.fillHeight: true + Layout.margins: 6 + cellWidth: Math.max(110, Math.floor(width / Math.max(1, Math.floor(width / 150)))) + cellHeight: 120 + clip: true + model: session.media + delegate: Item { + width: mediaGrid.cellWidth + height: mediaGrid.cellHeight + readonly property bool chosen: index === session.selectedMedia + ColumnLayout { + anchors.fill: parent + anchors.margins: 5 + spacing: 4 + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 72 + radius: 4 + color: root.viewerBackground + border.width: chosen ? 2 : 0 + border.color: root.accent + Image { + anchors.fill: parent + anchors.margins: chosen ? 2 : 0 + fillMode: Image.PreserveAspectFit + source: modelData.thumbnail + asynchronous: true + } + } + Text { Layout.fillWidth: true; text: modelData.name; color: root.fg; font.pixelSize: 10; elide: Text.ElideMiddle } + Text { text: root.timecode(modelData.duration); color: root.muted; font.pixelSize: 10 } + } + MouseArea { + anchors.fill: parent + onClicked: session.selectMedia(index) + onDoubleClicked: actions.append.trigger() + } + } + Text { + anchors.centerIn: parent + width: parent.width - 24 + visible: session.media.length === 0 + horizontalAlignment: Text.AlignHCenter + wrapMode: Text.WordWrap + text: "Import videos" + root.shortcutText(actions.importMedia) + color: root.muted + font.pixelSize: 11 + } + } + } + } + Rectangle { anchors.right: parent.right; width: 1; height: parent.height; color: root.line } + } + + ColumnLayout { + id: viewerColumn + x: root.compact || root.viewerOnly ? 0 : library.width + width: upper.width - x + height: upper.height + spacing: 0 + + // Adjustments bar (§6): only what applies to the selection is enabled. + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 44 + visible: !root.viewerOnly + color: colors.background + RowLayout { + anchors.fill: parent + anchors.leftMargin: 8 + anchors.rightMargin: 8 + spacing: 2 + OmaButton { iconName: "color"; text: "Color"; showLabel: !root.compact; enabled: false; tip: "Color (v0.1)" } + OmaButton { iconName: "crop"; text: "Crop"; showLabel: !root.compact; enabled: false; tip: "Crop and framing (v0.1)" } + OmaButton { + action: actions.volume + iconName: "volume" + showLabel: !root.compact + selected: root.drawer === "volume" + activeDot: root.audioAdjusted + } + OmaButton { iconName: "speed"; text: "Speed"; showLabel: !root.compact; enabled: false; tip: "Speed (v0.2)" } + OmaButton { iconName: "effects"; text: "Effects"; showLabel: !root.compact; enabled: false; tip: "Effects (v0.2)" } + OmaButton { iconName: "overlay"; text: "Overlay"; showLabel: !root.compact; enabled: false; tip: "Overlay, for layers above the storyline (v0.2)" } + OmaButton { + action: actions.info + iconName: "info" + showLabel: !root.compact + selected: root.drawer === "info" + } + Item { Layout.fillWidth: true } + OmaButton { + visible: root.compact + action: actions.toggleLibrary + iconName: "library" + showLabel: !root.narrow + selected: root.libraryOverlay + } + } + Rectangle { anchors.bottom: parent.bottom; width: parent.width; height: 1; color: root.line } + } + + // Drawer (§6): opens between the bar and the viewer, pushing the viewer down. + Rectangle { + Layout.fillWidth: true + Layout.preferredHeight: 64 + visible: !root.viewerOnly && root.drawer === "info" && !!session.info.name + color: colors.dark_background + GridLayout { + anchors.fill: parent + anchors.margins: 10 + columns: root.compact ? 2 : 4 + columnSpacing: 22 + rowSpacing: 4 + Field { label: "NAME"; value: session.info.name || "" } + Field { label: "DURATION"; value: root.timecode(session.info.duration || 0) } + Field { label: "FORMAT"; value: (session.info.codec || "") + " " + (session.info.resolution || "") + " " + (session.info.frameRate || "") } + Field { label: "DECODE PATH"; value: session.info.decodePath || "" } + } + Rectangle { anchors.bottom: parent.bottom; width: parent.width; height: 1; color: root.line } + } + + // Volume drawer (§6, level 1): volume, fades and mute for the selected clip. + // Values follow the clip; a change is committed as one command on release. + Rectangle { + id: volumeDrawer + Layout.fillWidth: true + Layout.preferredHeight: 64 + visible: !root.viewerOnly && root.drawer === "volume" && actions.volume.enabled + color: colors.dark_background + function commit() { + session.setClipAudio(gainControl.slider.value, fadeInControl.slider.value, + fadeOutControl.slider.value, !!session.info.muted) + } + component DrawerSlider: ColumnLayout { + property alias slider: control + property string label + property string readout + spacing: 0 + Layout.fillWidth: true + RowLayout { + Text { text: parent.parent.label; color: root.muted; font.pixelSize: 9; font.bold: true } + Item { Layout.fillWidth: true } + Text { text: parent.parent.readout; color: root.fg; font.pixelSize: 10 } + } + Slider { + id: control + Layout.fillWidth: true + focusPolicy: Qt.NoFocus + onPressedChanged: if (!pressed) volumeDrawer.commit() + } + } + RowLayout { + anchors.fill: parent + anchors.leftMargin: 12 + anchors.rightMargin: 12 + spacing: 18 + DrawerSlider { + id: gainControl + label: "VOLUME" + readout: root.decibels(gainControl.slider.value) + } + DrawerSlider { + id: fadeInControl + label: "FADE IN" + readout: fadeInControl.slider.value.toFixed(2) + " s" + } + DrawerSlider { + id: fadeOutControl + label: "FADE OUT" + readout: fadeOutControl.slider.value.toFixed(2) + " s" + } + OmaButton { + text: session.info.muted ? "Unmute" : "Mute" + selected: !!session.info.muted + onClicked: session.setClipAudio(gainControl.slider.value, fadeInControl.slider.value, + fadeOutControl.slider.value, !session.info.muted) + } + } + // The sliders follow the selected clip, except while one is being dragged. + Binding { target: gainControl.slider; property: "from"; value: 0 } + Binding { target: gainControl.slider; property: "to"; value: 2 } + Binding { target: gainControl.slider; property: "value"; value: session.info.gain || 0; when: !gainControl.slider.pressed } + Binding { target: fadeInControl.slider; property: "to"; value: Math.max(0.01, session.info.clipDuration || 0) } + Binding { target: fadeInControl.slider; property: "value"; value: session.info.fadeIn || 0; when: !fadeInControl.slider.pressed } + Binding { target: fadeOutControl.slider; property: "to"; value: Math.max(0.01, session.info.clipDuration || 0) } + Binding { target: fadeOutControl.slider; property: "value"; value: session.info.fadeOut || 0; when: !fadeOutControl.slider.pressed } + Rectangle { anchors.bottom: parent.bottom; width: parent.width; height: 1; color: root.line } + } + + // Viewer (§5): neutral background, untouched by the theme. + Rectangle { + Layout.fillWidth: true + Layout.fillHeight: true + color: root.viewerBackground + PreviewItem { objectName: "preview"; anchors.fill: parent } + Text { + anchors.centerIn: parent + visible: !session.hasMedia && !session.failed + text: session.media.length ? "Opening…" : "Import a video to start" + root.shortcutText(actions.importMedia) + color: "#8f9095" + font.pixelSize: 12 + } + // Playback status: nothing when all is well, the reason when it fails. + Rectangle { + visible: session.failed + anchors.horizontalCenter: parent.horizontalCenter + anchors.bottom: parent.bottom + anchors.bottomMargin: 12 + width: Math.min(parent.width - 24, failure.implicitWidth + 24) + height: 30 + radius: 6 + color: "#2a1d1f" + Text { + id: failure + anchors.centerIn: parent + width: parent.width - 24 + text: session.status + color: colors.red || "#ff8a8a" + font.pixelSize: 11 + elide: Text.ElideRight + } + } + } + + // Transport (§5): the same actions serve buttons and keyboard shortcuts. + Rectangle { + id: transport + Layout.fillWidth: true + Layout.preferredHeight: 42 + color: colors.background + RowLayout { + anchors.fill: parent + anchors.leftMargin: 12 + anchors.rightMargin: 8 + spacing: 2 + Text { + Layout.preferredWidth: root.narrow ? 104 : 214 + text: root.timecode(session.position) + (root.narrow ? "" : " / " + root.timecode(session.duration)) + color: session.hasMedia ? root.fg : root.muted + font.pixelSize: 12 + } + OmaButton { objectName: "transportStart"; action: actions.toStart; iconName: "start"; showLabel: false } + OmaButton { action: actions.previousFrame; iconName: "previous"; showLabel: false } + OmaButton { action: actions.playBackward; iconName: "backward"; showLabel: false } + OmaButton { objectName: "transportStop"; action: actions.stop; iconName: "stop"; showLabel: false } + OmaButton { + action: actions.playPause + iconName: session.playing ? "pause" : "play" + showLabel: false + implicitWidth: 44 + } + OmaButton { action: actions.playForward; iconName: "forward"; showLabel: false } + OmaButton { action: actions.nextFrame; iconName: "next"; showLabel: false } + OmaButton { objectName: "transportEnd"; action: actions.toEnd; iconName: "end"; showLabel: false } + Text { // shuttle speed (J/K/L), shown only when not normal + Layout.preferredWidth: 34 + visible: session.speed !== 0 && session.speed !== 1 + text: (session.speed < 0 ? "−" : "") + Math.abs(session.speed) + "×" + color: root.accent + font.pixelSize: 12 + font.bold: true + } + Item { + Layout.preferredWidth: 1 + Layout.fillWidth: true + implicitHeight: 32 + OmaButton { + anchors.right: parent.right + action: root.viewerOnly ? actions.leaveFullViewer : actions.fullViewer + iconName: "fullscreen" + showLabel: false + selected: root.viewerOnly + } + } + } + Rectangle { anchors.top: parent.top; width: parent.width; height: 1; color: root.line } + } + } + } + + // The single divider between the upper area and the timeline (§2.1). + Rectangle { + id: splitter + visible: !root.viewerOnly + anchors.top: upper.bottom + width: parent.width + height: 6 + color: colors.background + Rectangle { width: 36; height: 2; radius: 1; color: root.muted; anchors.centerIn: parent; opacity: 0.6 } + MouseArea { + anchors.fill: parent + cursorShape: Qt.SplitVCursor + onPositionChanged: (mouse) => { + if (pressed) + root.upperRatio = Math.max(0.35, Math.min(0.75, mapToItem(workspace, mouse.x, mouse.y).y / workspace.height)) + } + } + } + + // Timeline (§7): edit actions, minimap, storyline, and zoom. + Rectangle { + id: timelinePanel + visible: !root.viewerOnly + anchors.top: splitter.bottom + anchors.bottom: parent.bottom + width: parent.width + color: colors.darker_background + + readonly property real total: session.duration + readonly property real scale: root.fitTimeline && total > 0 + ? Math.min(2400, Math.max(4, (timelineScroll.width - 48) / total)) : root.pixelsPerSecond + readonly property real origin: 24 // left margin of time zero, in content pixels + + Rectangle { + id: editBar + anchors.top: parent.top + width: parent.width + height: 42 + color: colors.dark_background + RowLayout { + anchors.fill: parent + anchors.leftMargin: 12 + anchors.rightMargin: 12 + spacing: 3 + OmaButton { action: actions.append; iconName: "append"; showLabel: !root.compact } + OmaButton { action: actions.insert; iconName: "insert"; showLabel: !root.compact } + OmaButton { action: actions.overwrite; iconName: "overwrite"; showLabel: !root.compact } + Separator { Layout.leftMargin: 5; Layout.rightMargin: 5 } + OmaButton { objectName: "editSplit"; action: actions.split; iconName: "split"; showLabel: !root.compact } + OmaButton { action: actions.remove; iconName: "rippleDelete"; showLabel: !root.compact } + OmaButton { action: actions.lift; iconName: "lift"; showLabel: !root.compact } + Item { Layout.fillWidth: true } + } + Rectangle { anchors.bottom: parent.bottom; width: parent.width; height: 1; color: root.line } + } + + // Minimap: the whole project, with the visible region highlighted. + Item { + id: minimap + anchors.top: editBar.bottom + anchors.left: parent.left + anchors.right: parent.right + anchors.margins: 12 + anchors.leftMargin: 24 + anchors.rightMargin: 24 + height: 14 + visible: session.clips.length > 0 + Repeater { + model: session.clips + delegate: Rectangle { + x: minimap.width * modelData.start / Math.max(0.001, timelinePanel.total) + width: Math.max(2, minimap.width * modelData.duration / Math.max(0.001, timelinePanel.total) - 1) + anchors.verticalCenter: parent.verticalCenter + height: 6 + radius: 2 + color: modelData.id === session.selectedClip ? root.accent : colors.blue + opacity: 0.85 + } + } + Rectangle { // playhead + x: minimap.width * session.position / Math.max(0.001, timelinePanel.total) + width: 2 + height: parent.height + color: root.accent + } + Rectangle { // visible region + readonly property real content: Math.max(1, timelineScroll.contentWidth) + x: minimap.width * timelineScroll.contentX / content + width: Math.max(8, minimap.width * Math.min(1, timelineScroll.width / content)) + height: parent.height + radius: 3 + color: "transparent" + border.color: root.fg + opacity: 0.35 + } + MouseArea { + anchors.fill: parent + onPressed: (mouse) => navigate(mouse.x) + onPositionChanged: (mouse) => { if (pressed) navigate(mouse.x) } + function navigate(x) { + const target = x / minimap.width * timelineScroll.contentWidth - timelineScroll.width / 2 + timelineScroll.contentX = Math.max(0, Math.min(timelineScroll.contentWidth - timelineScroll.width, target)) + } + } + } + + Flickable { + id: timelineScroll + anchors.top: minimap.bottom + anchors.topMargin: 10 + anchors.bottom: zoomBar.top + anchors.left: parent.left + anchors.right: parent.right + clip: true + interactive: false // dragging belongs to scrubbing and trimming; scroll with the wheel or minimap + boundsBehavior: Flickable.StopAtBounds + contentWidth: Math.max(width, timelinePanel.origin * 2 + timelinePanel.total * timelinePanel.scale) + contentHeight: height + function secondsAt(x) { return Math.max(0, (x - timelinePanel.origin) / timelinePanel.scale) } + WheelHandler { + acceptedModifiers: Qt.ControlModifier + onWheel: (event) => event.angleDelta.y > 0 ? actions.zoomIn.trigger() : actions.zoomOut.trigger() + } + WheelHandler { + acceptedModifiers: Qt.NoModifier + onWheel: (event) => { + const delta = event.angleDelta.x !== 0 ? event.angleDelta.x : event.angleDelta.y + timelineScroll.contentX = Math.max(0, Math.min(timelineScroll.contentWidth - timelineScroll.width, + timelineScroll.contentX - delta)) + } + } + // Clicking or dragging on empty timeline moves the playhead (scrubbing). + MouseArea { + width: timelineScroll.contentWidth + height: timelineScroll.height + onPressed: (mouse) => session.seek(timelineScroll.secondsAt(mouse.x)) + onPositionChanged: (mouse) => { if (pressed) session.seek(timelineScroll.secondsAt(mouse.x)) } + } + Repeater { + model: session.clips + delegate: Rectangle { + id: clipItem + readonly property bool chosen: modelData.id === session.selectedClip + // Live feedback while an edge is dragged; committed as one trim on release. + property real headDrag: 0 + property real tailDrag: 0 + x: timelinePanel.origin + modelData.start * timelinePanel.scale + y: 4 + width: Math.max(6, modelData.duration * timelinePanel.scale - 2 - headDrag + tailDrag) + height: Math.min(86, Math.max(44, timelineScroll.height - 12)) + radius: 5 + clip: true + color: Qt.tint(colors.lighter_background, Qt.rgba(0.31, 0.55, 1, 0.18)) + border.width: chosen ? 2 : 0 + border.color: root.accent + Image { + anchors.fill: parent + anchors.margins: 2 + anchors.bottomMargin: 20 + source: modelData.thumbnail + fillMode: Image.TileHorizontally + verticalAlignment: Image.AlignVCenter + asynchronous: true + sourceSize.height: height + } + // Fades: a ramp over the clip's ends, as long as the fade. + Shape { + id: fades + anchors.fill: parent + visible: modelData.fadeIn > 0 || modelData.fadeOut > 0 + preferredRendererType: Shape.CurveRenderer + readonly property real fadeInWidth: modelData.fadeIn * timelinePanel.scale + readonly property real fadeOutWidth: modelData.fadeOut * timelinePanel.scale + ShapePath { + strokeColor: "transparent" + fillColor: Qt.rgba(0, 0, 0, 0.45) + startX: 0; startY: 0 + PathLine { x: fades.fadeInWidth; y: 0 } + PathLine { x: 0; y: clipItem.height } + PathLine { x: 0; y: 0 } + } + ShapePath { + strokeColor: "transparent" + fillColor: Qt.rgba(0, 0, 0, 0.45) + startX: clipItem.width; startY: 0 + PathLine { x: clipItem.width - fades.fadeOutWidth; y: 0 } + PathLine { x: clipItem.width; y: clipItem.height } + PathLine { x: clipItem.width; y: 0 } + } + } + Text { + anchors.left: parent.left + anchors.right: parent.right + anchors.bottom: parent.bottom + anchors.margins: 6 + anchors.bottomMargin: 4 + text: modelData.name + color: root.fg + font.pixelSize: 10 + elide: Text.ElideRight + } + MouseArea { + anchors.fill: parent + // Selects the clip and moves the playhead to the click. + onPressed: (mouse) => { + session.selectClip(modelData.id) + session.seek(modelData.start + mouse.x / timelinePanel.scale) + } + } + // Edges: drag to trim. The storyline is magnetic, so later clips follow + // (ripple trim, ui-design §7.3). + component Edge: MouseArea { + property bool head + property real pressX: 0 + width: 8 + height: parent.height + cursorShape: Qt.SizeHorCursor + hoverEnabled: true + preventStealing: true + Rectangle { + anchors.fill: parent + anchors.margins: 1 + radius: 3 + color: root.accent + opacity: parent.containsMouse || parent.pressed ? 0.9 : 0 + } + onPressed: (mouse) => { pressX = mapToItem(timelineScroll.contentItem, mouse.x, 0).x } + onPositionChanged: (mouse) => { + if (!pressed) return + const dx = mapToItem(timelineScroll.contentItem, mouse.x, 0).x - pressX + if (head) clipItem.headDrag = dx + else clipItem.tailDrag = dx + } + onReleased: { + const dx = head ? clipItem.headDrag : clipItem.tailDrag + clipItem.headDrag = 0 + clipItem.tailDrag = 0 + const frames = Math.round(dx / timelinePanel.scale * session.frameRate) + if (frames !== 0) session.trimClip(modelData.id, head, frames) + } + } + Edge { head: true; anchors.left: parent.left } + Edge { head: false; anchors.right: parent.right } + } + } + // The playhead across the timeline. + Rectangle { + id: playhead + visible: session.hasMedia + x: timelinePanel.origin + session.position * timelinePanel.scale - 1 + width: 2 + height: timelineScroll.height + color: root.accent + Rectangle { width: 10; height: 10; radius: 5; x: -4; y: -2; color: root.accent } + // Keep the playhead in view while playing. + onXChanged: { + if (!session.playing) return + if (x > timelineScroll.contentX + timelineScroll.width - 40 || x < timelineScroll.contentX) + timelineScroll.contentX = Math.max(0, Math.min(timelineScroll.contentWidth - timelineScroll.width, x - 40)) + } + } + Text { + visible: session.clips.length === 0 + x: 24 + y: 12 + text: "Add clips with " + actions.append.keys + " or by double-clicking them in the library." + color: root.muted + font.pixelSize: 11 + } + } + + RowLayout { + id: zoomBar + anchors.bottom: parent.bottom + anchors.right: parent.right + anchors.rightMargin: 12 + height: 36 + spacing: 2 + OmaButton { action: actions.zoomOut; iconName: "zoomOut"; showLabel: false } + Slider { + Layout.preferredWidth: 110 + from: Math.log(4) + to: Math.log(2400) + value: Math.log(timelinePanel.scale) + focusPolicy: Qt.NoFocus + onMoved: { root.fitTimeline = false; root.pixelsPerSecond = Math.exp(value) } + } + OmaButton { action: actions.zoomIn; iconName: "zoomIn"; showLabel: false } + OmaButton { action: actions.zoomFit; iconName: "fit"; showLabel: false; selected: root.fitTimeline } + } + } + } + } + + // Every action's shortcut, registered in one place. + Repeater { + model: [actions.playPause, actions.pause, actions.stop, actions.playForward, actions.playBackward, actions.previousFrame, + actions.nextFrame, actions.back10, actions.forward10, actions.toStart, actions.toEnd, + actions.append, actions.insert, actions.overwrite, actions.split, actions.remove, + actions.lift, actions.undo, actions.redo, actions.importMedia, + actions.exportMovie, actions.toggleLibrary, actions.fullViewer, actions.leaveFullViewer, + actions.zoomIn, actions.zoomOut, actions.zoomFit, actions.volume, actions.info] + delegate: Item { + Shortcut { + sequence: modelData.keys + enabled: modelData.enabled && (session.editing || modelData === actions.importMedia) + onActivated: modelData.trigger() + } + } + } +} diff --git a/apps/omamovie/qml/OmaAction.qml b/apps/omamovie/qml/OmaAction.qml new file mode 100644 index 0000000..adae03b --- /dev/null +++ b/apps/omamovie/qml/OmaAction.qml @@ -0,0 +1,8 @@ +import QtQuick.Controls + +// An editor command (ui-design §8.1). `keys` is its default shortcut; Main.qml registers every +// action's keys once. A button that uses the action does not register them again, because two +// identical sequences are ambiguous in Qt and neither fires. +Action { + property string keys +} diff --git a/apps/omamovie/qml/OmaButton.qml b/apps/omamovie/qml/OmaButton.qml new file mode 100644 index 0000000..7720b27 --- /dev/null +++ b/apps/omamovie/qml/OmaButton.qml @@ -0,0 +1,61 @@ +import QtQuick +import QtQuick.Controls + +// A flat button showing an icon, a label or both; tooltips carry the name and shortcut. +AbstractButton { + id: control + property string iconName + property bool showLabel: true + property bool primary: false + property bool selected: false + property bool activeDot: false + property string tip: action ? action.text : text + focusPolicy: Qt.NoFocus // Space and letters belong to the editor's actions + hoverEnabled: true + implicitHeight: 32 + implicitWidth: content.implicitWidth + (showLabel ? 22 : 14) + ToolTip.visible: hovered && (!showLabel || iconName === "") && tip !== "" + ToolTip.text: tip + (action && action.keys ? " (" + action.keys + ")" : "") + ToolTip.delay: 500 + background: Rectangle { + radius: 6 + color: control.primary && control.enabled ? colors.accent + : control.selected ? colors.selection_background + : control.down || (control.hovered && control.enabled) ? colors.lighter_background + : "transparent" + } + contentItem: Item { + implicitWidth: content.implicitWidth + implicitHeight: content.implicitHeight + Row { + id: content + anchors.centerIn: parent + spacing: 6 + readonly property color tint: !control.enabled ? colors.dark_foreground + : control.primary ? colors.background + : control.selected ? colors.accent : colors.foreground + Icon { + visible: control.iconName !== "" + name: control.iconName + color: content.tint + opacity: control.enabled ? 1 : 0.55 + anchors.verticalCenter: parent.verticalCenter + } + Text { + visible: control.showLabel && control.text !== "" + text: control.text + color: content.tint + opacity: control.enabled ? 1 : 0.55 + font.pixelSize: 12 + anchors.verticalCenter: parent.verticalCenter + } + } + Rectangle { // marks an adjustment already active on the clip (ui-design §6) + visible: control.activeDot + width: 5; height: 5; radius: 2.5 + color: colors.accent + anchors.right: parent.right + anchors.top: parent.top + } + } +} diff --git a/apps/omamovie/qml/Separator.qml b/apps/omamovie/qml/Separator.qml new file mode 100644 index 0000000..1c36b11 --- /dev/null +++ b/apps/omamovie/qml/Separator.qml @@ -0,0 +1,3 @@ +import QtQuick + +Rectangle { implicitWidth: 1; implicitHeight: 20; color: colors.lighter_background } diff --git a/apps/omamovie/src/audio_player.cpp b/apps/omamovie/src/audio_player.cpp new file mode 100644 index 0000000..6af4fdd --- /dev/null +++ b/apps/omamovie/src/audio_player.cpp @@ -0,0 +1,170 @@ +#include "audio_player.hpp" + +#include "timeline_audio.hpp" + +#include "oma/base/log.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t kBlockFrames = 1024; +// Audio rendered ahead of the device. Enough to ride out a slow decode; small enough that a +// seek restarts quickly (the ring is flushed anyway). +constexpr double kAheadSeconds = 0.25; + +} // namespace + +AudioPlayer::AudioPlayer(oma::SampleRate rate) : rate_(rate), clock_(rate) {} + +AudioPlayer::~AudioPlayer() { + stop(); + pipeline_.shutdown(); +} + +oma::Result AudioPlayer::ensure_output() { + if (output_) { + return {}; + } + const oma::audio::OutputFormat format{.rate = rate_, .channels = kChannels}; + if (!silent_) { + if (auto pipewire = oma::audio::make_pipewire_output(format)) { + output_ = std::move(*pipewire); + on_device_ = true; + return {}; + } else { + // Degrade, never fail silently (CLAUDE.md §19): playback keeps an audio-paced clock. + oma::log_warn(oma::Category::Audio, "no audio device, playing silently: {}", + pipewire.error().summary()); + } + } + auto null = oma::audio::make_null_output(format); + if (!null) { + return std::unexpected(null.error()); + } + output_ = std::move(*null); + on_device_ = false; + return {}; +} + +oma::Result AudioPlayer::start(oma::timeline::Timeline timeline, + std::unordered_map paths, std::int64_t from) { + stop(); + if (output_ && !output_->healthy()) { + // The device went away: open a new output (PipeWire picks the current default sink). + oma::log_warn(oma::Category::Audio, "audio output lost its device; reopening"); + live_output_.store(nullptr, std::memory_order_release); + output_->stop(); + retired_.push_back(std::move(output_)); + } + if (auto r = ensure_output(); !r) { + return r; + } + output_->set_muted(muted_); + if (auto r = clock_.anchor(rate_.sample_to_time(from), output_->frames_consumed()); !r) { + return r; + } + anchor_consumed_.store(output_->frames_consumed(), std::memory_order_relaxed); + anchor_sample_.store(clock_.anchor_sample(), std::memory_order_relaxed); + live_output_.store(output_.get(), std::memory_order_release); + render_failed_.store(false, std::memory_order_relaxed); + render_error_.clear(); + auto renderer = std::make_shared(std::move(timeline), std::move(paths), rate_, kChannels); + oma::audio::SampleRing& ring = output_->ring(); + const auto ahead = static_cast(kAheadSeconds * rate_.hz()); + producer_ = pipeline_.submit("playback-audio", [this, renderer, &ring, ahead, from](oma::JobContext& ctx) { + std::vector block(kBlockFrames * static_cast(kChannels)); + std::int64_t next = from; + bool reported = false; + while (!ctx.is_cancelled()) { + const std::size_t queued = ring.readable(); + if (const int stall = stall_ms_.exchange(0, std::memory_order_relaxed); stall > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(stall)); + continue; + } + if (queued >= ahead || ring.writable() < kBlockFrames) { + std::this_thread::sleep_for(std::chrono::milliseconds(4)); + continue; + } + if (auto r = renderer->render(block, next); !r && !reported) { + reported = true; + render_error_ = r.error().summary(); + render_failed_.store(true, std::memory_order_release); + } + float peak = 0.0F; + for (const float v : block) peak = std::max(peak, std::abs(v)); + if (peak > peak_.load(std::memory_order_relaxed)) peak_.store(peak, std::memory_order_relaxed); + ring.write(block); + next += static_cast(kBlockFrames); + } + return oma::Result{}; + }); + // The device may request its first quantum as soon as it becomes active. Give the + // producer time to decode two blocks before enabling that real-time callback. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(500); + while (ring.readable() < 2 * kBlockFrames && + std::chrono::steady_clock::now() < deadline && + !render_failed_.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + if (auto r = output_->start(); !r) { + stop(); + return r; + } + running_ = true; + return {}; +} + +void AudioPlayer::stop() noexcept { + // Order matters for the single-producer/single-consumer ring: the producer is joined first, + // then the device itself drops the queued frames (a flush on an active output runs in its + // callback, which on PipeWire's data thread may still be running), and only then the device + // stops pulling. + if (producer_.valid()) { + producer_.cancel(); + (void)producer_.wait(); + producer_ = {}; + } + if (output_) { + output_->flush(); + output_->stop(); + } + running_ = false; +} + +std::int64_t AudioPlayer::audible_sample() const noexcept { + const oma::audio::AudioOutput* out = live_output_.load(std::memory_order_acquire); + if (out == nullptr) { + return anchor_sample_.load(std::memory_order_relaxed); + } + // PlaybackClock's rule (oma/audio/clock.hpp), on the atomic copies of its anchor. + const std::int64_t played = out->frames_consumed() - anchor_consumed_.load(std::memory_order_relaxed) - + out->latency_frames(); + return anchor_sample_.load(std::memory_order_relaxed) + std::max(played, 0); +} + +bool AudioPlayer::healthy() const noexcept { + return !output_ || output_->healthy(); +} + +void AudioPlayer::set_muted(bool muted) noexcept { + muted_ = muted; + if (output_) output_->set_muted(muted); +} + +std::int64_t AudioPlayer::underruns() const noexcept { + return output_ ? output_->underruns() : 0; +} + +std::optional AudioPlayer::take_error() { + if (!render_failed_.exchange(false, std::memory_order_acquire)) { + return std::nullopt; + } + return render_error_; +} diff --git a/apps/omamovie/src/audio_player.hpp b/apps/omamovie/src/audio_player.hpp new file mode 100644 index 0000000..2cb6dc7 --- /dev/null +++ b/apps/omamovie/src/audio_player.hpp @@ -0,0 +1,89 @@ +#pragma once + +#include "oma/audio/clock.hpp" +#include "oma/audio/output.hpp" +#include "oma/base/error.hpp" +#include "oma/base/jobs.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include +#include +#include + +// Plays the sequence's audio and provides the playback master clock (CLAUDE.md §12): a pipeline +// thread renders timeline audio into the output's lock-free ring, the device consumes it, and +// the audible position comes from the frames the device actually consumed minus its latency. +// Video follows that position, dropping or repeating frames; audio is never stretched. +// +// Threading: start/stop/position are called on the UI thread. The pipeline runs on its own +// one-thread JobPool (the "playback" pipeline of ADR-0003) and is the ring's only producer; the +// device callback is its only consumer. stop() cancels and joins the pipeline before the ring +// is flushed, so the two never write concurrently. +class AudioPlayer { +public: + static constexpr int kChannels = 2; + + explicit AudioPlayer(oma::SampleRate rate); + ~AudioPlayer(); + AudioPlayer(const AudioPlayer&) = delete; + AudioPlayer& operator=(const AudioPlayer&) = delete; + AudioPlayer(AudioPlayer&&) = delete; + AudioPlayer& operator=(AudioPlayer&&) = delete; + + // Starts playing `timeline` from sequence sample `from`. Media IDs map to files in `paths`. + [[nodiscard]] oma::Result start(oma::timeline::Timeline timeline, + std::unordered_map paths, + std::int64_t from); + void stop() noexcept; + [[nodiscard]] bool running() const noexcept { return running_; } + // Plays into the silent null output even when a device exists (automated runs). + void set_silent(bool silent) noexcept { silent_ = silent; } + // Highest absolute sample rendered since the last call (a level meter; also how tests see + // that clip audio is actually mixed). + [[nodiscard]] float take_peak() noexcept { return peak_.exchange(0.0F, std::memory_order_relaxed); } + + // The sequence sample audible now. It holds still until the device plays real samples. + // Safe from any thread (the viewer's presentation measurement reads it on the render + // thread); during a restart it may briefly report the previous anchor. + [[nodiscard]] std::int64_t audible_sample() const noexcept; + // False when the device stream failed or lost its device; the caller restarts playback, + // which opens a new output. + [[nodiscard]] bool healthy() const noexcept; + // Device-level mute for measurements: the device clock keeps running. + void set_muted(bool muted) noexcept; + // Diagnostics: the pipeline stops producing for `ms`, forcing underruns. + void inject_stall(int ms) noexcept { stall_ms_.store(ms, std::memory_order_relaxed); } + // Whether sound reaches a device (PipeWire) rather than the silent null output. + [[nodiscard]] bool on_device() const noexcept { return on_device_; } + [[nodiscard]] std::int64_t underruns() const noexcept; + // Rendering failures (undecodable media) since start, reported once per start. + [[nodiscard]] std::optional take_error(); + +private: + [[nodiscard]] oma::Result ensure_output(); + + oma::SampleRate rate_; + std::unique_ptr output_; + oma::audio::PlaybackClock clock_; + bool on_device_ = false; + bool silent_ = false; + std::atomic peak_{0.0F}; + bool muted_ = false; + std::atomic stall_ms_{0}; + // Mirrors of the clock's anchor for lock-free reads from other threads. + std::atomic anchor_sample_{0}; + std::atomic anchor_consumed_{0}; + std::atomic live_output_{nullptr}; + // Outputs replaced after losing their device stay alive until the player goes: another + // thread may still be reading the one it loaded. Replacement is rare (device loss). + std::vector> retired_; + bool running_ = false; + std::atomic render_failed_{false}; + std::string render_error_; // written by the pipeline before setting render_failed_ + oma::JobHandle producer_; + oma::JobPool pipeline_{1}; // destroyed first: the producer never outlives the output +}; diff --git a/apps/omamovie/src/frame_source.cpp b/apps/omamovie/src/frame_source.cpp new file mode 100644 index 0000000..ae69d91 --- /dev/null +++ b/apps/omamovie/src/frame_source.cpp @@ -0,0 +1,112 @@ +#include "frame_source.hpp" + +#include +#include +#include +#include +#include + +namespace { + +oma::RationalTime pts_of(const oma::media::VideoFrame& frame) { + return frame.pts().value_or(oma::RationalTime{}); +} + +// How far ahead decoding forward beats seeking (which restarts at the previous keyframe). +constexpr double kForwardWindowSeconds = 2.0; + +} // namespace + +oma::Result FrameSource::stream(const std::string& path) { + const auto it = std::ranges::find(streams_, path, &Stream::path); + if (it != streams_.end()) { + streams_.splice(streams_.begin(), streams_, it); + return &streams_.front(); + } + oma::media::VideoDecoderOptions options; + options.paths = {oma::media::DecodePath::Software}; + auto decoder = oma::media::VideoDecoder::open(std::filesystem::path(path), options); + if (!decoder) { + return std::unexpected(decoder.error()); + } + if (streams_.size() >= kMaxStreams) { + streams_.pop_back(); + } + streams_.push_front( + Stream{.path = path, .decoder = std::move(*decoder), .current = {}, .ahead = {}, .ended = false}); + return &streams_.front(); +} + +oma::Result FrameSource::seek(Stream& s, const oma::RationalTime& t) { + if (auto r = s.decoder->seek(t); !r) { + return r; + } + auto first = s.decoder->next(); + if (!first) { + return std::unexpected(first.error()); + } + if (!*first) { + return oma::make_error(oma::ErrorCode::OutOfRange, oma::Category::Playback, "no frame at this time", + s.path); + } + s.current = std::make_shared(std::move(**first)); + auto next = s.decoder->next(); + if (!next) { + return std::unexpected(next.error()); + } + s.ahead = std::move(*next); + s.ended = !s.ahead; + return {}; +} + +oma::Result FrameSource::advance(Stream& s, const oma::RationalTime& t) { + while (s.ahead && pts_of(*s.ahead) <= t) { + s.current = std::make_shared(std::move(*s.ahead)); + auto next = s.decoder->next(); + if (!next) { + return std::unexpected(next.error()); + } + s.ahead = std::move(*next); + s.ended = !s.ahead; + } + return {}; +} + +oma::Result FrameSource::picture_at(const std::string& path, const oma::RationalTime& t) { + auto opened = stream(path); + if (!opened) { + return std::unexpected(opened.error()); + } + Stream& s = **opened; + const bool from_current = + s.current && pts_of(*s.current) <= t && + (t.seconds_approx() - pts_of(*s.current).seconds_approx()) <= kForwardWindowSeconds; + if (auto r = from_current ? advance(s, t) : seek(s, t); !r) { + return std::unexpected(r.error()); + } + Picture picture{.frame = s.current, .color = {}, .rotation = 0, .sample_aspect = oma::Rational::literal(1, 1)}; + if (const auto& video = s.decoder->stream().video) { + picture.color = video->color; + picture.rotation = video->rotation; + picture.sample_aspect = video->sample_aspect; + } + return picture; +} + +oma::Result FrameSource::image_at(const std::string& path, const oma::RationalTime& t) { + auto picture = picture_at(path, t); + if (!picture) { + return std::unexpected(picture.error()); + } + const oma::media::VideoFrame& frame = *picture->frame; + QImage image(frame.width(), frame.height(), QImage::Format_RGBA8888); + if (image.isNull()) { + return oma::make_error(oma::ErrorCode::Internal, oma::Category::Ui, "cannot allocate the thumbnail"); + } + auto copied = + frame.copy_rgba({image.bits(), static_cast(image.sizeInBytes())}, image.bytesPerLine()); + if (!copied) { + return std::unexpected(copied.error()); + } + return image; +} diff --git a/apps/omamovie/src/frame_source.hpp b/apps/omamovie/src/frame_source.hpp new file mode 100644 index 0000000..d0fa059 --- /dev/null +++ b/apps/omamovie/src/frame_source.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/rational.hpp" +#include "oma/base/time.hpp" +#include "oma/media/probe.hpp" +#include "oma/media/video_decoder.hpp" + +#include + +#include +#include +#include +#include +#include + +// A decoded picture and how to interpret it, ready to be a compositor layer input. The frame is +// shared: the decoder keeps it as its current frame while the viewer composites it. +struct Picture { + std::shared_ptr frame; + oma::media::ColorInfo color; + int rotation = 0; // counterclockwise display rotation of the stream + oma::Rational sample_aspect = oma::Rational::literal(1, 1); +}; + +// Decodes the frame on screen at a media time, for the viewer and thumbnails. Software decode +// for now: the viewer's compositor uploads the planes once (CLAUDE.md §7.3 fallback). Hardware +// frames need the playback scheduler's queue admission first (ADR-0005). +// +// Threading: owned by the session's single job worker; every call happens on that thread +// (jobs on a one-thread JobPool never overlap). Pictures handed out are only read elsewhere. +class FrameSource { +public: + // The frame whose PTS is the last one <= `t` (the first frame when `t` precedes it). + // Moving forward a little reuses the open decoder without seeking, so playback decodes + // each frame once. + [[nodiscard]] oma::Result picture_at(const std::string& path, const oma::RationalTime& t); + + // The same frame converted to RGBA8 on the CPU (thumbnails, not the viewer). + [[nodiscard]] oma::Result image_at(const std::string& path, const oma::RationalTime& t); + +private: + struct Stream { + std::string path; + std::unique_ptr decoder; + std::shared_ptr current; // last frame returned (pts <= target) + std::optional ahead; // the frame after it, if decoded + bool ended = false; // no frame after `ahead`/`current` + }; + + [[nodiscard]] oma::Result stream(const std::string& path); + [[nodiscard]] static oma::Result seek(Stream& s, const oma::RationalTime& t); + [[nodiscard]] static oma::Result advance(Stream& s, const oma::RationalTime& t); + + // A few open decoders, most recently used first: cutting between clips of the same files + // keeps their decoders warm. Bounded so memory does not grow with the library. + static constexpr std::size_t kMaxStreams = 4; + std::list streams_; +}; diff --git a/apps/omamovie/src/main.cpp b/apps/omamovie/src/main.cpp new file mode 100644 index 0000000..57026c2 --- /dev/null +++ b/apps/omamovie/src/main.cpp @@ -0,0 +1,495 @@ +// OmaMovie editor shell: Qt Quick on OmaMovie's Vulkan device (ADR-0005), the M5 timeline +// through Session, and a software-frame viewer until M4's GPU handoff and audio clock. +// +// QtTest drives the --smoke keyboard check. With Qt 6.11 its headers complete classes such as +// QBitArray after a standard-library SFINAE probe saw them incomplete, which GCC 16 reports +// (-Wsfinae-incomplete); the diagnostic is about Qt's header order, not this file. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsfinae-incomplete" +#endif +#include +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#include "preview_item.hpp" +#include "session.hpp" + +#include "oma/gpu/device.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +QVariantMap currentTheme() { + QVariantMap colors{ + {"background", "#111318"}, {"dark_background", "#171a20"}, + {"darker_background", "#0b0d11"}, {"lighter_background", "#262a33"}, + {"foreground", "#e2e5ee"}, {"dark_foreground", "#8b91a2"}, + {"accent", "#8ea2ff"}, {"selection_background", "#26314a"}, + {"blue", "#4f8dff"}, {"green", "#4dffa6"}, {"yellow", "#ffd75f"}, + {"red", "#ff6b6b"}, {"mode", "dark"}}; + QProcess process; + process.start(QStringLiteral("omarchy-theme-color"), {QStringLiteral("--all")}); + if (!process.waitForFinished(1000) || process.exitCode() != 0) return colors; + for (const QByteArray& line : process.readAllStandardOutput().split('\n')) { + const auto parts = line.split('\t'); + if (parts.size() != 2) continue; + const QString key = QString::fromUtf8(parts[0]); + const QString value = QString::fromUtf8(parts[1]).trimmed(); + if (colors.contains(key) && (value.startsWith('#') || key == "mode")) + colors.insert(key, value); + } + return colors; +} + +QFont currentFont() { + QFont fallback = QFontDatabase::systemFont(QFontDatabase::FixedFont); + QProcess process; + process.start(QStringLiteral("omarchy-font-current")); + if (!process.waitForFinished(1000) || process.exitCode() != 0) return fallback; + const QString family = QString::fromUtf8(process.readAllStandardOutput()).trimmed(); + return family.isEmpty() ? fallback : QFont(family); +} + +// --smoke: drives the editor through import, edits, undo/redo, playback and keyboard +// shortcuts, saving screenshots when OMA_GUI_SMOKE_*_SCREENSHOT name files. Each step waits for +// its condition (imports and frames are asynchronous) up to a deadline. +class SmokeRun : public QObject { +public: + struct Step { + const char* name; + std::function ready; + std::function act; + }; + + SmokeRun(QGuiApplication& app, std::vector steps) : app_(app), steps_(std::move(steps)) { + connect(&timer_, &QTimer::timeout, this, [this] { poll(); }); + timer_.start(50); + since_.start(); + } + + // Milliseconds since the current step began, for steps that just need time to pass. + [[nodiscard]] qint64 elapsed() const { return since_.elapsed(); } + +private: + void poll() { + if (index_ >= steps_.size()) return; + Step& step = steps_[index_]; + if (!step.ready()) { + if (since_.elapsed() > 8000) { + std::printf("GUI smoke: FAIL (timed out at '%s')\n", step.name); + timer_.stop(); + app_.exit(1); + } + return; + } + step.act(); + ++index_; + since_.restart(); + } + + QGuiApplication& app_; + std::vector steps_; + std::size_t index_ = 0; + QTimer timer_; + QElapsedTimer since_; +}; + +void screenshot(QQuickWindow* window, const char* variable) { + const QString path = qEnvironmentVariable(variable); + if (!path.isEmpty()) window->grabWindow().save(path); +} + +} // namespace + +int main(int argc, char** argv) { + QGuiApplication application(argc, argv); + application.setFont(currentFont()); + QQuickWindow::setGraphicsApi(QSGRendererInterface::Vulkan); + const bool smoke = argc == 2 && std::string_view(argv[1]) == "--smoke"; + const bool audit = argc == 4 && std::string_view(argv[1]) == "--m4-audit"; + bool valid_seconds = false; + const int audit_seconds = audit ? QString::fromLocal8Bit(argv[3]).toInt(&valid_seconds) : 0; + if (!(argc == 1 || smoke || audit || (argc == 2 && argv[1][0] != '-')) || + (audit && (!valid_seconds || audit_seconds < 5 || audit_seconds > 3600))) { + std::fprintf(stderr, "usage: %s [video-file | --smoke | --m4-audit video-file seconds]\n", argv[0]); + return 2; + } + oma::gpu::DeviceOptions options; + options.internally_synchronized_queues = false; + options.instance_extensions = {VK_KHR_SURFACE_EXTENSION_NAME, "VK_KHR_wayland_surface", + VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME}; + auto created = oma::gpu::Device::create(options); + if (!created) { + std::fprintf(stderr, "%s\n", created.error().summary().c_str()); + return 1; + } + auto device = std::move(*created); + QVulkanInstance instance; + instance.setVkInstance(device->instance()); + instance.setApiVersion(QVersionNumber(1, 4)); + if (!instance.create()) { + std::fprintf(stderr, "Qt could not adopt the Vulkan instance\n"); + return 1; + } + qmlRegisterType("OmaMovie", 1, 0, "PreviewItem"); + Session session; + QQmlApplicationEngine engine; + engine.rootContext()->setContextProperty("session", &session); + engine.rootContext()->setContextProperty("colors", currentTheme()); + engine.load(QUrl::fromLocalFile( + QFileInfo(QStringLiteral("apps/omamovie/qml/Main.qml")).absoluteFilePath())); + if (engine.rootObjects().isEmpty()) return 1; + auto* window = qobject_cast(engine.rootObjects().first()); + auto* preview = window ? window->findChild("preview") : nullptr; + if (!window || !preview) { + std::fprintf(stderr, "OmaMovie QML did not create the preview window\n"); + return 1; + } + preview->setDevice(device.get()); + session.setPreview(preview); + window->setVulkanInstance(&instance); + window->setGraphicsDevice(QQuickGraphicsDevice::fromDeviceObjects( + device->physical_device(), device->device(), device->graphics_family(), 0)); + QObject::connect(window, &QQuickWindow::beforeFrameBegin, window, + [&] { device->lock_queue(device->graphics_family(), 0); }, + Qt::DirectConnection); + QObject::connect(window, &QQuickWindow::afterFrameEnd, window, + [&] { device->unlock_queue(device->graphics_family(), 0); }, + Qt::DirectConnection); + window->show(); + if (audit) { + // Internal timing diagnostic: the output remains paced by PipeWire (or NullOutput), + // but its samples are muted. The video timestamp is the last completed composition, + // which precedes Qt's actual presentation. + session.setOutputMuted(true); + session.open(QString::fromLocal8Bit(argv[2])); + QElapsedTimer waiting; + waiting.start(); + QElapsedTimer elapsed; + QTimer timer; + std::vector errors; + double min_signed = std::numeric_limits::infinity(); + double max_signed = -std::numeric_limits::infinity(); + double first_sum = 0.0; + double last_sum = 0.0; + std::size_t first_count = 0; + std::size_t last_count = 0; + std::int64_t initial_underruns = 0; + double first_underrun_at = -1.0; + bool started = false; + bool settled = false; + QObject::connect(&timer, &QTimer::timeout, &application, [&] { + if (session.failed()) { + std::fprintf(stderr, "M4 audit: %s\n", qPrintable(session.status())); + application.exit(1); + return; + } + if (!started) { + if (!session.hasMedia()) { + if (waiting.elapsed() > 15000) application.exit(1); + return; + } + if (session.duration() < static_cast(audit_seconds)) { + std::fprintf(stderr, "M4 audit: media is shorter than %d seconds\n", audit_seconds); + application.exit(2); + return; + } + session.seek(0); + session.togglePlay(); + started = true; + elapsed.start(); + return; + } + const double seconds = static_cast(elapsed.elapsed()) / 1000.0; + if (seconds >= 2.0 && !settled) { + initial_underruns = session.audioUnderruns(); + settled = true; + } + if (settled && first_underrun_at < 0.0 && + session.audioUnderruns() > initial_underruns) first_underrun_at = seconds; + if (seconds >= 2.0 && seconds < static_cast(audit_seconds) && + preview->lastCompositedFrame() >= 0) { + const double audio_time = static_cast(session.audibleSample()) / 48000.0; + const double video_time = static_cast(preview->lastCompositedFrame()) / + session.frameRate(); + const double signed_error = audio_time - video_time; + min_signed = std::min(min_signed, signed_error); + max_signed = std::max(max_signed, signed_error); + const double error = std::abs(signed_error); + errors.push_back(error); + if (seconds < 62.0) { first_sum += signed_error; ++first_count; } + if (seconds >= static_cast(audit_seconds - 60)) { + last_sum += signed_error; + ++last_count; + } + } + if (seconds < static_cast(audit_seconds)) return; + if (errors.empty()) { + std::fprintf(stderr, "M4 audit: no synchronized frames\n"); + application.exit(1); + return; + } + std::ranges::sort(errors); + const double p99 = errors[errors.size() * 99 / 100]; + const double maximum = errors.back(); + const double drift = first_count > 0 && last_count > 0 + ? std::abs(first_sum / static_cast(first_count) - + last_sum / static_cast(last_count)) + : 0.0; + const auto underruns = session.audioUnderruns() - initial_underruns; + session.pause(); + const double frame_ms = 1000.0 / session.frameRate(); + const bool passed = p99 * 1000.0 <= frame_ms && + (audit_seconds < 120 || drift * 1000.0 <= frame_ms) && + underruns == 0; + std::printf("M4 audit: %s, %d s, %s audio, %zu samples, %u composites, " + "p99 %.2f ms, max %.2f ms, first/last mean delta %.2f ms, " + "signed range %.2f..%.2f ms, underruns %lld (first %.2f s), dropped video %lld, " + "frame interval %.2f ms\n", + passed ? "PASS" : "FAIL", audit_seconds, + session.audioOnDevice() ? "PipeWire" : "null", errors.size(), + preview->presentedFrames(), p99 * 1000.0, maximum * 1000.0, + drift * 1000.0, min_signed * 1000.0, max_signed * 1000.0, + static_cast(underruns), first_underrun_at, + static_cast(session.droppedVideoFrames()), frame_ms); + application.exit(passed ? 0 : 1); + }); + timer.start(10); + return application.exec(); + } + if (!smoke) { + if (argc == 2) session.open(QString::fromLocal8Bit(argv[1])); + return application.exec(); + } + + window->resize(1600, 900); + session.setSilent(true); + struct Results { + bool imported = false, split = false, undone = false, redone = false, rippled = false; + bool inserted = false, overwritten = false, trimmed = false, audio = false; + bool volume = false, muted = false, gpu_viewer = false, shuttle = false; + bool played = false, space = false, escape = false, stepped = false, keys_focused = true; + bool controls = false; + double after_steps = 0; + } r; + const auto clip_count = [&] { return session.clips().size(); }; + // Keys count only while the window keeps the focus: Qt fires window shortcuts in the + // active window, and the desktop may move the focus during the run. Synthetic modifier + // chords are unreliable on Wayland (resolved against the real keyboard), so only plain + // keys are pressed. + const auto press = [&](Qt::Key key) { + r.keys_focused = r.keys_focused && QGuiApplication::focusWindow() == window; + QTest::keyClick(window, key); + }; + const auto click_button = [&](const char* name) { + auto* button = window->findChild(QString::fromLatin1(name)); + if (!button || !button->isVisible() || !button->isEnabled()) { + std::printf("GUI smoke: unavailable button %s\n", name); + return false; + } + const QPointF center = button->mapToScene(QPointF(button->width() / 2, button->height() / 2)); + QTest::mouseClick(window, Qt::LeftButton, Qt::NoModifier, center.toPoint()); + return true; + }; + std::unique_ptr run; + const auto after = [&](qint64 ms) { return [&run, ms] { return run->elapsed() >= ms; }; }; + run = std::make_unique(application, std::vector{ + {"start", after(300), [&] { + screenshot(window, "OMA_GUI_SMOKE_PROJECTS_SCREENSHOT"); + session.newProject(); + session.open(QStringLiteral("tests/fixtures/generated/h264_30fps_aac.mp4")); + }}, + {"first import", [&] { return clip_count() == 1; }, [&] { + session.open(QStringLiteral("tests/fixtures/generated/hevc_10bit.mp4")); + }}, + {"second import", [&] { return clip_count() == 2; }, [&] { + r.imported = std::abs(session.duration() - 2.0) < 1e-6; + session.seek(0.5); + }}, + {"split", after(400), [&] { + screenshot(window, "OMA_GUI_SMOKE_WIDE_SCREENSHOT"); + // The viewer composited on the GPU and Qt drew it: the center of the viewer shows + // the video (the fixture's test pattern), not the neutral background. + const QImage grab = window->grabWindow(); + const QPointF center = preview->mapToScene(QPointF(preview->width() / 2, preview->height() / 2)); + const QColor pixel = grab.pixelColor(static_cast(center.x() * grab.devicePixelRatio()), + static_cast(center.y() * grab.devicePixelRatio())); + r.gpu_viewer = preview->presentedFrames() > 0 && pixel != QColor(0x1b, 0x1b, 0x1d) && + pixel != QColor(Qt::black); + if (!r.gpu_viewer) + std::printf("GUI smoke: viewer presented %u frames, center %s\n", preview->presentedFrames(), + qPrintable(pixel.name())); + session.splitAtPlayhead(); + r.split = clip_count() == 3 && session.undoText() == QStringLiteral("Split"); + session.undo(); + r.undone = clip_count() == 2 && session.canRedo(); + session.redo(); + r.redone = clip_count() == 3; + // The left half stays selected: ripple delete closes the gap. + session.deleteSelected(true); + r.rippled = clip_count() == 2 && std::abs(session.duration() - 1.5) < 1e-6; + // The library's first video (1 s) at the playhead, 0.5 s into the sequence. + session.selectMedia(0); + session.seek(0.5); + session.insertSelected(); + r.inserted = clip_count() == 3 && std::abs(session.duration() - 2.5) < 1e-6; + session.undo(); + session.overwriteSelected(); // covers the second clip exactly + r.overwritten = clip_count() == 2 && std::abs(session.duration() - 1.5) < 1e-6; + session.undo(); + const double first = session.clips().front().toMap().value("id").toDouble(); + session.trimClip(first, true, 5); // ripple: the sequence loses 5 frames + r.trimmed = std::abs(session.duration() - (1.5 - 5.0 / 30.0)) < 1e-6; + session.undo(); + r.trimmed = r.trimmed && std::abs(session.duration() - 1.5) < 1e-6; + session.seek(0); + (void)session.takeAudioPeak(); + session.togglePlay(); + }}, + {"play", after(700), [&] { + r.played = session.position() > 0.2; + // The 440 Hz fixture tone (amplitude ~0.125) must reach the mix, paced by audio. + r.audio = session.audioRunning() && session.takeAudioPeak() > 0.05F; + session.pause(); + session.seek(0); + }}, + {"shuttle forward", after(300), [&] { + // L twice: double speed. Half a second of wall time covers about a second. + session.pause(); + session.seek(0); + session.shuttle(1); + session.shuttle(1); + }}, + {"shuttle check", after(450), [&] { + const double forward = session.position(); + r.shuttle = session.speed() == 2 && forward > 0.6; + session.pause(); + session.seek(1.2); + session.shuttle(-1); // J: reverse at normal speed + }}, + {"reverse check", after(400), [&] { + r.shuttle = r.shuttle && session.speed() == -1 && session.position() < 1.1; + session.pause(); // K + r.shuttle = r.shuttle && !session.playing(); + if (!r.shuttle) std::printf("GUI smoke: shuttle failed at %.3fs\n", session.position()); + }}, + {"volume", after(100), [&] { + // The first clip carries the fixture's tone. Fades snap to whole frames. + session.selectClip(session.clips().front().toMap().value("id").toDouble()); + session.setClipAudio(0.5, 0.11, 0.0, false); + const QVariantMap info = session.info(); + r.volume = session.undoText() == QStringLiteral("Audio Adjustments") && + std::abs(info.value("gain").toDouble() - 0.5) < 1e-6 && + std::abs(info.value("fadeIn").toDouble() - 3.0 / 30.0) < 1e-9; + window->setProperty("drawer", QStringLiteral("volume")); + session.setClipAudio(0.5, 0.11, 0.0, true); // mute: the mix must go silent + session.seek(0); + (void)session.takeAudioPeak(); + session.togglePlay(); + }}, + {"muted play", after(500), [&] { + r.muted = session.audioRunning() && session.takeAudioPeak() == 0.0F; + session.pause(); + screenshot(window, "OMA_GUI_SMOKE_VOLUME_SCREENSHOT"); + window->setProperty("drawer", QString()); + session.undo(); + session.undo(); + session.seek(0); + window->requestActivate(); + }}, + {"space", after(300), [&] { press(Qt::Key_Space); }}, + {"space again", after(250), [&] { + r.space = session.playing(); + press(Qt::Key_Space); + }}, + {"escape", after(250), [&] { + r.space = r.space && !session.playing(); + window->setProperty("viewerOnly", true); + press(Qt::Key_Escape); + }}, + {"frames", after(250), [&] { + r.escape = !window->property("viewerOnly").toBool(); + session.seek(0); + for (int i = 0; i < 3; ++i) press(Qt::Key_Right); + }}, + {"half width", after(400), [&] { + r.after_steps = session.position(); + r.stepped = std::abs(r.after_steps - 3.0 / session.frameRate()) < 1e-9; + window->resize(820, 620); + }}, + {"classic controls", after(400), [&] { + session.seek(0.25); + const int before = clip_count(); + const bool split_clicked = click_button("editSplit"); + const bool split = split_clicked && clip_count() == before + 1; + if (split) session.undo(); + session.seek(0.5); + session.togglePlay(); + const bool stop_clicked = click_button("transportStop"); + const bool stopped = stop_clicked && !session.playing() && session.position() == 0.0; + const bool end_clicked = click_button("transportEnd"); + const bool at_end = end_clicked && session.position() > 0.0; + const bool start_clicked = click_button("transportStart"); + r.controls = split && stopped && at_end && start_clicked && session.position() == 0.0; + }}, + {"library", after(500), [&] { + window->setProperty("libraryOverlay", true); + }}, + {"finish", after(300), [&] { + screenshot(window, "OMA_GUI_SMOKE_LIBRARY_SCREENSHOT"); + window->setProperty("libraryOverlay", false); + screenshot(window, "OMA_GUI_SMOKE_SCREENSHOT"); + const bool edits = r.imported && r.split && r.undone && r.redone && r.rippled && r.inserted && + r.overwritten && r.trimmed && r.played && r.audio && r.volume && r.muted && + r.gpu_viewer && r.shuttle; + const bool keys = r.space && r.escape && r.stepped; + std::printf("GUI smoke: edits %s (import %d, split %d, undo %d, redo %d, ripple delete %d, " + "insert %d, overwrite %d, trim %d, play %d, audio %d, volume %d, mute %d, " + "GPU viewer %d, J/K/L %d)\n", + edits ? "PASS" : "FAIL", r.imported, r.split, r.undone, r.redone, r.rippled, + r.inserted, r.overwritten, r.trimmed, r.played, r.audio, r.volume, r.muted, + r.gpu_viewer, r.shuttle); + if (r.keys_focused) { + std::printf("GUI smoke: keyboard %s (Space %d, Escape %d, Right x3 -> %.4fs)\n", + keys ? "PASS" : "FAIL", r.space, r.escape, r.after_steps); + } else { + std::printf("GUI smoke: keyboard skipped (window not focused throughout)\n"); + } + std::printf("GUI smoke: classic controls %s\n", r.controls ? "PASS" : "FAIL"); + const bool passed = edits && r.controls && (keys || !r.keys_focused) && !session.failed(); + if (session.failed()) std::printf("GUI smoke: error: %s\n", qPrintable(session.status())); + std::printf("GUI smoke: %s\n", passed ? "PASS" : "FAIL"); + application.exit(passed ? 0 : 1); + }}, + }); + return application.exec(); +} diff --git a/apps/omamovie/src/preview_item.cpp b/apps/omamovie/src/preview_item.cpp new file mode 100644 index 0000000..98d6d93 --- /dev/null +++ b/apps/omamovie/src/preview_item.cpp @@ -0,0 +1,140 @@ +#include "preview_item.hpp" + +#include "oma/base/log.hpp" +#include "oma/compositor/compositor.hpp" +#include "oma/gpu/device.hpp" +#include "oma/gpu/resources.hpp" + +#include +#include +#include +#include + +#include + +#include + +PreviewItem::PreviewItem(QQuickItem* parent) : QQuickItem(parent) { + setFlag(ItemHasContents); + connect(this, &QQuickItem::windowChanged, this, [this](QQuickWindow* window) { + if (window == nullptr) return; + // Before Qt tears its device state down, on the render thread. + connect(window, &QQuickWindow::sceneGraphInvalidated, this, [this] { releaseGpu(); }, + Qt::DirectConnection); + }); +} + +PreviewItem::~PreviewItem() { + // Normally released with the scene graph; this covers an item destroyed on its own. + if (compositor_) releaseGpu(); +} + +void PreviewItem::setFrame(std::shared_ptr frame) { + frame_ = std::move(frame); + ++generation_; + update(); +} + +void PreviewItem::waitQueueIdle() const { + // vkQueueWaitIdle needs the queue externally synchronized; the lock is recursive, so this + // also works inside the frame the Qt bridge holds. + device_->lock_queue(device_->graphics_family(), 0); + vkQueueWaitIdle(device_->queue(device_->graphics_family(), 0)); + device_->unlock_queue(device_->graphics_family(), 0); +} + +void PreviewItem::releaseGpu() { + if (device_ == nullptr || !compositor_) return; + // Nothing may still sample or write the images being destroyed. + waitQueueIdle(); + compositor_.reset(); + display_ = nullptr; + wrapped_ = nullptr; + rhi_texture_ = nullptr; + composited_ = 0; +} + +bool PreviewItem::composite() { + if (!compositor_) { + auto created = oma::compositor::VulkanCompositor::create(*device_); + if (!created) { + oma::log_error(oma::Category::Compositor, "viewer compositor: {}", created.error().summary()); + return false; + } + compositor_ = std::move(*created); + } + const ViewerFrame& f = *frame_; + if ((width_ != 0 && width_ != f.graph.width) || (height_ != 0 && height_ != f.graph.height)) { + // The display image is about to be replaced: Qt may still sample the old one. + waitQueueIdle(); + } + std::vector inputs; + inputs.reserve(f.pictures.size()); + for (const Picture& p : f.pictures) { + inputs.push_back( + {.frame = p.frame.get(), .color = p.color, .rotation = p.rotation, .sample_aspect = p.sample_aspect}); + } + if (auto r = compositor_->render(f.graph, inputs); !r) { + oma::log_error(oma::Category::Compositor, "viewer render: {}", r.error().summary()); + return false; + } + auto encoded = compositor_->encode_display(); + if (!encoded) { + oma::log_error(oma::Category::Compositor, "viewer display transform: {}", encoded.error().summary()); + return false; + } + width_ = f.graph.width; + height_ = f.graph.height; + display_ = reinterpret_cast((*encoded)->handle()); // NOLINT(performance-no-int-to-ptr) + composited_ = generation_; + last_composited_frame_.store(f.frame, std::memory_order_release); + presented_.fetch_add(1, std::memory_order_relaxed); + return true; +} + +QSGNode* PreviewItem::updatePaintNode(QSGNode* old, UpdatePaintNodeData* /*data*/) { + auto* node = static_cast(old); + const auto drop = [&]() -> QSGNode* { + delete node; // and with it the texture wrapper + wrapped_ = nullptr; + rhi_texture_ = nullptr; + return nullptr; + }; + if (!frame_ || device_ == nullptr) return drop(); + const bool encoded = composited_ != generation_; + if (encoded && !composite()) return drop(); + if (display_ == nullptr || window()->rhi() == nullptr) return drop(); + if (node == nullptr) { + node = new QSGSimpleTextureNode; + node->setOwnsTexture(true); + wrapped_ = nullptr; + } + if (wrapped_ != display_) { + const auto image = reinterpret_cast(display_); + auto* texture = window()->rhi()->newTexture(QRhiTexture::RGBA8, + QSize(static_cast(width_), static_cast(height_))); + if (!texture->createFrom({image, VK_IMAGE_LAYOUT_GENERAL})) { + delete texture; + oma::log_error(oma::Category::Compositor, "Qt could not wrap the viewer image"); + return drop(); + } + // The wrapper takes the QRhiTexture; the node takes the wrapper (and deletes the one it + // replaces). The VkImage stays the compositor's. + QSGTexture* wrapper = window()->createTextureFromRhiTexture(texture, QQuickWindow::TextureOwnsGLTexture); + if (wrapper == nullptr) { + delete texture; + return drop(); + } + wrapper->setFiltering(QSGTexture::Linear); + node->setTexture(wrapper); + rhi_texture_ = texture; + wrapped_ = display_; + } else if (encoded) { + node->markDirty(QSGNode::DirtyMaterial); + } + // The encode left the image in GENERAL; Qt transitions from the layout it is told. + if (encoded) rhi_texture_->setNativeLayout(VK_IMAGE_LAYOUT_GENERAL); + const QSizeF fit = QSizeF(width_, height_).scaled(boundingRect().size(), Qt::KeepAspectRatio); + node->setRect((width() - fit.width()) / 2, (height() - fit.height()) / 2, fit.width(), fit.height()); + return node; +} diff --git a/apps/omamovie/src/preview_item.hpp b/apps/omamovie/src/preview_item.hpp new file mode 100644 index 0000000..6f602ea --- /dev/null +++ b/apps/omamovie/src/preview_item.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include "viewer_frame.hpp" + +#include + +#include +#include +#include +#include + +class QRhiTexture; +class QSGTexture; + +namespace oma::gpu { +class Device; +} +namespace oma::compositor { +class VulkanCompositor; +} + +// The viewer (ui-design §5). Composites the frame with the Vulkan compositor on Qt's render +// thread and hands the result to the scene graph as a texture over the same VkImage: no +// readback, no CPU conversion (ADR-0005, CLAUDE.md §9.4). +// +// Threading and the shared queue: compositing happens in updatePaintNode, on the render thread, +// while the GUI thread is blocked and while the Qt bridge holds the graphics queue for the frame +// (the queue lock is recursive, so the compositor's own submission re-enters it). Because the +// same thread creates, resizes and destroys the swapchain, compositor submissions can never +// overlap Qt's vkDeviceWaitIdle there. Ordering on the one queue does the rest: the next encode +// waits for Qt's earlier sampling, and Qt's sampling comes after the encode. +class PreviewItem : public QQuickItem { + Q_OBJECT +public: + explicit PreviewItem(QQuickItem* parent = nullptr); + ~PreviewItem() override; + PreviewItem(const PreviewItem&) = delete; + PreviewItem& operator=(const PreviewItem&) = delete; + PreviewItem(PreviewItem&&) = delete; + PreviewItem& operator=(PreviewItem&&) = delete; + + // OmaMovie's device, the one Qt renders with. Set once, before the first frame. + void setDevice(const oma::gpu::Device* device) { device_ = device; } + // GUI thread. nullptr shows the neutral viewer background. + void setFrame(std::shared_ptr frame); + + // Frames composited and handed to Qt (diagnostics, the smoke check). + [[nodiscard]] unsigned presentedFrames() const { return presented_; } + [[nodiscard]] std::int64_t lastCompositedFrame() const { + return last_composited_frame_.load(std::memory_order_acquire); + } + +protected: + QSGNode* updatePaintNode(QSGNode* old, UpdatePaintNodeData* data) override; + +private: + bool composite(); // render thread + void releaseGpu(); // render thread: waits for the queue, then drops the GPU objects + void waitQueueIdle() const; + + const oma::gpu::Device* device_ = nullptr; + std::shared_ptr frame_; // written on the GUI thread, read during sync + unsigned generation_ = 0; + std::atomic presented_{0}; + std::atomic last_composited_frame_{-1}; + + // Render thread only. The scene-graph node owns the texture wrapper (and through it the + // QRhiTexture borrowing the compositor's display image), so Qt releases its own objects; + // the item owns only the compositor. + unsigned composited_ = 0; + std::unique_ptr compositor_; + const void* display_ = nullptr; // the VkImage of the last encode + const void* wrapped_ = nullptr; // the VkImage the node's texture wraps + QRhiTexture* rhi_texture_ = nullptr; // owned by the node's QSGTexture + std::uint32_t width_ = 0; + std::uint32_t height_ = 0; +}; diff --git a/apps/omamovie/src/session.cpp b/apps/omamovie/src/session.cpp new file mode 100644 index 0000000..f63c0a9 --- /dev/null +++ b/apps/omamovie/src/session.cpp @@ -0,0 +1,669 @@ +#include "session.hpp" + +#include "preview_item.hpp" + +#include "oma/compositor/render_graph.hpp" +#include "oma/timeline/edit.hpp" +#include "oma/timeline/evaluate.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace tl = oma::timeline; + +namespace { + +constexpr int kNoticeMs = 3500; +constexpr std::int64_t kStillSeconds = 4; // default length of a picture on the storyline +const auto kSequenceAudioRate = oma::SampleRate::make(48000).value(); + +QString message(const oma::Error& error) { + return QString::fromStdString(error.message()); +} + +} // namespace + +Session::Session(QObject* parent) + : Session(std::make_unique(), parent) {} + +Session::Session(std::unique_ptr importer, QObject* parent) + : QObject(parent), + importer_(importer ? std::move(importer) : std::make_unique()), + audio_(kSequenceAudioRate) { + notice_timer_.setSingleShot(true); + connect(¬ice_timer_, &QTimer::timeout, this, [this] { + notice_.clear(); + emit statusChanged(); + }); + tick_.setTimerType(Qt::PreciseTimer); + connect(&tick_, &QTimer::timeout, this, &Session::onTick); +} + +Session::~Session() { + workers_.shutdown(); +} + +// ------------------------------------------------------------------- screens + +void Session::newProject() { + ++generation_; + pause(); + for (oma::JobHandle& h : imports_) h.cancel(); + imports_.clear(); + library_.clear(); + editor_.reset(); + snapshot_.reset(); + refreshPaths(); + primary_ = {}; + selected_clip_ = {}; + selected_media_ = -1; + playhead_ = 0; + wanted_.reset(); + failed_ = false; + status_.clear(); + notice_.clear(); + if (preview_ != nullptr) preview_->setFrame(nullptr); + editing_ = true; + emit libraryChanged(); + emit sequenceChanged(); + emit selectionChanged(); + emit positionChanged(); + emit statusChanged(); + emit viewChanged(); +} + +void Session::showProjects() { + pause(); + editing_ = false; + emit viewChanged(); +} + +void Session::continueProject() { + editing_ = true; + emit viewChanged(); +} + +// ------------------------------------------------------------------- import + +void Session::importUrl(const QUrl& url) { + importFile(url.toLocalFile(), !hasMedia()); +} + +void Session::importFile(const QString& path, bool append) { + if (path.isEmpty()) return; + if (!editing_) { + editing_ = true; + emit viewChanged(); + } + setNotice(QStringLiteral("Importing %1…").arg(QFileInfo(path).fileName())); + const unsigned generation = generation_; + const tl::MediaId id(next_media_++); + const QString thumbnail_path = + thumbnails_.filePath(QString::number(id.value()) + QStringLiteral(".png")); + std::erase_if(imports_, [](const oma::JobHandle& h) { + const auto s = h.state(); + return s != oma::JobState::Pending && s != oma::JobState::Running; + }); + imports_.push_back(workers_.submit("import", [this, generation, id, path, thumbnail_path, append](oma::JobContext&) { + const std::string file = path.toStdString(); + auto probed = importer_->inspect_input(std::filesystem::path(file)); + if (!probed || !probed->best_video) { + const QString why = probed ? QStringLiteral("no video stream") : message(probed.error()); + QMetaObject::invokeMethod(this, [this, generation, path, why] { + if (generation == generation_) + fail(QStringLiteral("Cannot import %1: %2").arg(QFileInfo(path).fileName(), why)); + }, Qt::QueuedConnection); + return oma::Result{}; + } + const auto& stream = probed->streams[static_cast(*probed->best_video)]; + const oma::Rational tb = stream.timebase; + LibraryItem item; + item.path = path; + item.name = QFileInfo(path).fileName(); + item.media.id = id; + item.media.start = stream.start.value_or(*oma::RationalTime::make(0, tb)); + const auto length = stream.duration ? stream.duration : probed->duration; + if (length) { + auto in_tb = length->rescaled(item.media.start.timebase(), oma::Rounding::Floor); + item.media.duration = in_tb.value_or(oma::RationalTime{}); + item.seconds = length->seconds_approx(); + } else { + item.media.duration = *oma::RationalTime::make(0, item.media.start.timebase()); + } + item.media.has_video = true; + item.media.has_audio = probed->best_audio.has_value(); + item.media.still = stream.video && stream.video->still_image; + if (item.media.still) item.seconds = static_cast(kStillSeconds); + item.details.insert("codec", QString::fromStdString(stream.codec)); + if (stream.video) { + item.rate = stream.video->frame_rate; + const bool sideways = stream.video->rotation % 180 != 0; + item.width = static_cast(sideways ? stream.video->height : stream.video->width); + item.height = static_cast(sideways ? stream.video->width : stream.video->height); + item.details.insert("resolution", + QStringLiteral("%1 × %2").arg(stream.video->width).arg(stream.video->height)); + if (item.rate) { + const double fps = item.rate->fps().to_double_approx(); + item.details.insert("frameRate", + QString::number(fps, 'f', fps == std::floor(fps) ? 0 : 3) + + (stream.video->variable_frame_rate ? QStringLiteral(" fps (variable)") + : QStringLiteral(" fps"))); + } + } + item.details.insert("decodePath", QStringLiteral("software (preview prototype)")); + if (auto first = frames_.image_at(file, item.media.start)) { + if (first->scaled(256, 144, Qt::KeepAspectRatio, Qt::SmoothTransformation).save(thumbnail_path)) + item.thumbnail = QUrl::fromLocalFile(thumbnail_path).toString(); + } + QMetaObject::invokeMethod(this, [this, generation, item = std::move(item), append]() mutable { + if (generation == generation_) addToLibrary(std::move(item), append); + }, Qt::QueuedConnection); + return oma::Result{}; + })); +} + +void Session::addToLibrary(LibraryItem item, bool append) { + if (editor_) { + if (auto r = editor_->add_media(item.media); !r) { + fail(message(r.error())); + return; + } + } + library_.push_back(std::move(item)); + refreshPaths(); + selected_media_ = static_cast(library_.size()) - 1; + notice_.clear(); + emit statusChanged(); + emit libraryChanged(); + emit selectionChanged(); + if (append) appendSelected(); +} + +const Session::LibraryItem* Session::item(tl::MediaId id) const { + const auto it = std::ranges::find_if(library_, [&](const LibraryItem& i) { return i.media.id == id; }); + return it == library_.end() ? nullptr : &*it; +} + +// The sequence takes its frame rate from the first clip placed on it (ui-design §3). +bool Session::ensureSequence(const LibraryItem& first) { + if (editor_) return true; + const oma::FrameRate rate = first.rate.value_or(oma::frame_rates::k30); + auto timeline = tl::Timeline::create(rate, kSequenceAudioRate); + if (!timeline) timeline = tl::Timeline::create(oma::frame_rates::k30, kSequenceAudioRate); + if (!timeline) { + fail(message(timeline.error())); + return false; + } + editor_.emplace(std::move(*timeline)); + if (first.width > 0 && first.height > 0) { + // Even sizes keep 4:2:0 exports possible later. + canvas_width_ = (first.width + 1) & ~1U; + canvas_height_ = (first.height + 1) & ~1U; + } + for (const LibraryItem& i : library_) { + if (auto r = editor_->add_media(i.media); !r) { + fail(message(r.error())); + editor_.reset(); + return false; + } + } + primary_ = editor_->new_track_id(); + if (auto r = editor_->execute(tl::edit::add_track(primary_, tl::TrackKind::Video, "Storyline")); !r) { + fail(message(r.error())); + editor_.reset(); + return false; + } + editor_->clear_history(); // the storyline is part of the new project, not an edit + refreshSnapshot(); + return true; +} + +std::optional Session::sourceFor(const LibraryItem& item) const { + const tl::Timeline& t = editor_->timeline(); + std::optional ticks; + if (item.media.still) { + ticks = oma::rescale(kStillSeconds, oma::Rational::literal(1, 1), t.timebase(), oma::Rounding::Floor).value_or(0); + } else { + // Whole frames of the sequence that fit in the media (rounded down: never past its end). + auto exact = oma::rescale(item.media.duration.value(), item.media.duration.timebase(), t.timebase(), + oma::Rounding::Floor); + if (exact) ticks = *exact / ticksPerFrame() * ticksPerFrame(); + } + if (!ticks || *ticks <= 0) return std::nullopt; + return tl::edit::ClipSource{.media = item.media.id, + .source_in = item.media.start, + .duration = t.at(*ticks), + .time_map = {}, + .video = {}, + .audio = {}}; +} + +// ------------------------------------------------------------------- selection + +void Session::selectMedia(int index) { + if (index < 0 || index >= static_cast(library_.size())) return; + selected_media_ = index; + emit selectionChanged(); +} + +void Session::selectClip(double id) { + selected_clip_ = tl::ClipId(static_cast(id)); + if (editor_) { + if (const tl::Clip* c = editor_->timeline().find_clip(selected_clip_)) { + const auto it = std::ranges::find_if(library_, [&](const LibraryItem& i) { return i.media.id == c->media; }); + if (it != library_.end()) selected_media_ = static_cast(it - library_.begin()); + } + } + emit selectionChanged(); +} + +// ------------------------------------------------------------------- edits + +bool Session::run(std::unique_ptr command) { + if (!editor_) return false; + pause(); // playback renders a snapshot of the timeline; an edit stops it + if (auto r = editor_->execute(std::move(command)); !r) { + setNotice(message(r.error())); + return false; + } + afterEdit(); + return true; +} + +void Session::afterEdit() { + refreshSnapshot(); + if (selected_clip_.valid() && editor_->timeline().find_clip(selected_clip_) == nullptr) { + selected_clip_ = {}; + emit selectionChanged(); + } + emit sequenceChanged(); + emit selectionChanged(); // the selected clip's properties may have changed + setFrame(std::min(frame(), lastFrame())); // the content under the playhead may have changed +} + +// 0 = append, 1 = insert at the playhead, 2 = overwrite at the playhead. +void Session::placeSelected(int how) { + if (selected_media_ < 0 || selected_media_ >= static_cast(library_.size())) return; + const LibraryItem& source = library_[static_cast(selected_media_)]; + if (!ensureSequence(source)) return; + const auto clip = sourceFor(source); + if (!clip) { + setNotice(QStringLiteral("%1 is too short for one frame").arg(source.name)); + return; + } + const tl::ClipId id = editor_->new_clip_id(); + const auto at = editor_->timeline().at(playhead_); + const bool done = how == 0 ? run(tl::edit::append(primary_, id, *clip)) + : how == 1 ? run(tl::edit::insert(primary_, id, at, *clip)) + : run(tl::edit::overwrite(primary_, id, at, *clip)); + if (done) { + selected_clip_ = id; + emit selectionChanged(); + } +} + +void Session::appendSelected() { + placeSelected(0); +} + +void Session::insertSelected() { + placeSelected(1); +} + +void Session::overwriteSelected() { + placeSelected(2); +} + +void Session::splitAtPlayhead() { + if (!editor_) return; + const tl::Clip* c = editor_->timeline().clip_at(primary_, playhead_); + if (c == nullptr || c->start_ticks() == playhead_) { + setNotice(QStringLiteral("Move the playhead inside a clip to split it")); + return; + } + const tl::ClipId left = c->id; + if (run(tl::edit::split(left, editor_->timeline().at(playhead_)))) { + selected_clip_ = left; + emit selectionChanged(); + } +} + +void Session::deleteSelected(bool ripple) { + if (!editor_) return; + tl::ClipId id = selected_clip_; + if (!id.valid()) { + const tl::Clip* c = editor_->timeline().clip_at(primary_, playhead_); + if (c == nullptr) return; + id = c->id; + } + run(ripple ? tl::edit::ripple_delete(id) : tl::edit::remove_clip(id)); +} + +void Session::trimClip(double id, bool head, int frames) { + if (!editor_ || frames == 0) return; + const tl::ClipId clip(static_cast(id)); + const tl::Clip* c = editor_->timeline().find_clip(clip); + if (c == nullptr) return; + const std::int64_t delta = static_cast(frames) * ticksPerFrame(); + const tl::Timeline& t = editor_->timeline(); + if (run(head ? tl::edit::trim_start(clip, t.at(c->start_ticks() + delta), true) + : tl::edit::trim_end(clip, t.at(c->end_ticks() + delta), true))) { + selected_clip_ = clip; + emit selectionChanged(); + } +} + +void Session::setClipAudio(double gain, double fadeIn, double fadeOut, bool muted) { + if (!editor_ || !selected_clip_.valid()) return; + const tl::Clip* c = editor_->timeline().find_clip(selected_clip_); + if (c == nullptr || !std::isfinite(gain) || !std::isfinite(fadeIn) || !std::isfinite(fadeOut)) return; + const std::int64_t frames = c->duration.value() / ticksPerFrame(); + const std::int64_t in = std::clamp(std::llround(fadeIn * frameRate()), 0, frames); + const std::int64_t out = std::clamp(std::llround(fadeOut * frameRate()), 0, frames - in); + tl::AudioProperties audio = c->audio; + audio.gain = static_cast(std::clamp(gain, 0.0, 4.0)); + audio.fade_in = editor_->timeline().at(in * ticksPerFrame()); + audio.fade_out = editor_->timeline().at(out * ticksPerFrame()); + audio.muted = muted; + if (audio == c->audio) return; + const tl::ClipId id = c->id; + run(tl::edit::set_audio(id, audio)); +} + +void Session::undo() { + if (!editor_ || !editor_->can_undo()) return; + pause(); + if (auto r = editor_->undo(); !r) setNotice(message(r.error())); + afterEdit(); +} + +void Session::redo() { + if (!editor_ || !editor_->can_redo()) return; + pause(); + if (auto r = editor_->redo(); !r) setNotice(message(r.error())); + afterEdit(); +} + +// ------------------------------------------------------------------- read-out for QML + +QVariantList Session::media() const { + QVariantList list; + for (const LibraryItem& i : library_) { + list.push_back(QVariantMap{{"id", static_cast(i.media.id.value())}, + {"name", i.name}, + {"duration", i.seconds}, + {"thumbnail", i.thumbnail}}); + } + return list; +} + +QVariantList Session::clips() const { + QVariantList list; + if (!editor_) return list; + const tl::Timeline& t = editor_->timeline(); + const tl::Track* track = t.find_track(primary_); + if (track == nullptr) return list; + for (const tl::Clip& c : track->clips) { + const LibraryItem* source = item(c.media); + list.push_back(QVariantMap{{"id", static_cast(c.id.value())}, + {"start", c.start.seconds_approx()}, + {"duration", c.duration.seconds_approx()}, + {"name", source != nullptr ? source->name : QString()}, + {"thumbnail", source != nullptr ? source->thumbnail : QString()}, + {"fadeIn", c.audio.fade_in.seconds_approx()}, + {"fadeOut", c.audio.fade_out.seconds_approx()}, + {"audioAdjusted", c.audio.muted || c.audio.gain != 1.0F || + c.audio.fade_in.value() != 0 || c.audio.fade_out.value() != 0}}); + } + return list; +} + +QVariantMap Session::info() const { + const LibraryItem* source = nullptr; + if (editor_ && selected_clip_.valid()) { + if (const tl::Clip* c = editor_->timeline().find_clip(selected_clip_)) source = item(c->media); + } + if (source == nullptr && selected_media_ >= 0 && selected_media_ < static_cast(library_.size())) + source = &library_[static_cast(selected_media_)]; + if (source == nullptr) return {}; + QVariantMap out = source->details; + out.insert("name", source->name); + out.insert("duration", source->seconds); + out.insert("hasAudio", source->media.has_audio); + if (editor_ && selected_clip_.valid()) { + if (const tl::Clip* c = editor_->timeline().find_clip(selected_clip_)) { + out.insert("clip", true); + out.insert("clipDuration", c->duration.seconds_approx()); + out.insert("gain", static_cast(c->audio.gain)); + out.insert("fadeIn", c->audio.fade_in.seconds_approx()); + out.insert("fadeOut", c->audio.fade_out.seconds_approx()); + out.insert("muted", c->audio.muted); + } + } + return out; +} + +bool Session::hasMedia() const { + return editor_ && editor_->timeline().duration().value() > 0; +} + +double Session::duration() const { + return editor_ ? editor_->timeline().duration().seconds_approx() : 0; +} + +double Session::frameRate() const { + return editor_ ? editor_->timeline().frame_rate().fps().to_double_approx() : 30; +} + +QString Session::undoText() const { + return editor_ ? QString::fromUtf8(editor_->undo_name()) : QString(); +} + +QString Session::redoText() const { + return editor_ ? QString::fromUtf8(editor_->redo_name()) : QString(); +} + +double Session::position() const { + return editor_ ? editor_->timeline().at(playhead_).seconds_approx() : 0; +} + +// ------------------------------------------------------------------- playhead + +std::int64_t Session::ticksPerFrame() const { + const tl::Timeline& t = editor_->timeline(); + return t.to_ticks(t.frame_rate().frame_to_time(1)).value_or(1); +} + +std::int64_t Session::frame() const { + return editor_ ? playhead_ / ticksPerFrame() : 0; +} + +// The last frame that shows something; the end of the sequence is exclusive. +std::int64_t Session::lastFrame() const { + if (!editor_) return 0; + const std::int64_t end = editor_->timeline().duration().value(); + return end > 0 ? (end - 1) / ticksPerFrame() : 0; +} + +void Session::setFrame(std::int64_t frame) { + if (!editor_) return; + playhead_ = std::clamp(frame, 0, lastFrame()) * ticksPerFrame(); + emit positionChanged(); + requestFrame(); +} + +void Session::seek(double seconds) { + if (!editor_ || !std::isfinite(seconds)) return; + // Display seconds to the nearest frame of the sequence grid. + setFrame(std::llround(seconds * frameRate())); + if (playing()) startPlayback(); // restart the clock, audio and decode-ahead from here +} + +void Session::stepFrames(int frames) { + pause(); + setFrame(frame() + frames); +} + +void Session::toEnd() { + pause(); + setFrame(lastFrame()); +} + +void Session::togglePlay() { + setSpeed(playing() ? 0 : 1); +} + +void Session::shuttle(int direction) { + if (direction > 0) { + setSpeed(speed_ <= 0 ? 1 : std::min(speed_ * 2, 4)); + } else if (direction < 0) { + setSpeed(speed_ >= 0 ? -1 : std::max(speed_ * 2, -4)); + } +} + +void Session::setSpeed(int speed) { + if (speed == 0) { + pause(); + return; + } + if (!hasMedia()) return; + // From an end, play back into the sequence. + if (speed > 0 && frame() >= lastFrame()) setFrame(0); + if (speed < 0 && frame() <= 0) setFrame(lastFrame()); + speed_ = speed; + startPlayback(); + tick_.start(std::max(4, static_cast(500.0 / frameRate()))); + emit positionChanged(); +} + +void Session::startPlayback() { + audio_.stop(); + scheduler_.stop(); + wanted_.reset(); + play_from_frame_ = frame(); + clock_.restart(); + if (!snapshot_) refreshSnapshot(); + if (speed_ == 1) { + // The playhead sits on the frame grid; audio starts at the sample at or before it. + if (auto r = audio_.start(*snapshot_, *paths_, playhead_ / ticksPerSample()); !r) { + setNotice(QStringLiteral("Playing without audio: %1").arg(message(r.error()))); + } else if (!audio_.on_device() && !warned_silent_) { + warned_silent_ = true; + setNotice(QStringLiteral("No audio device: playing silently")); + } + } + // At double and quadruple speed every second or fourth frame is shown. + scheduler_.start(snapshot_, paths_, canvas_width_, canvas_height_, ticksPerFrame(), frame(), speed_, lastFrame()); +} + +void Session::pause() { + if (speed_ == 0) return; + speed_ = 0; + tick_.stop(); + audio_.stop(); + scheduler_.stop(); + emit positionChanged(); + requestFrame(); // the exact frame at the playhead, whatever playback last showed +} + +std::int64_t Session::ticksPerSample() const { + const tl::Timeline& t = editor_->timeline(); + return std::max(1, t.to_ticks(kSequenceAudioRate.sample_to_time(1)).value_or(1)); +} + +void Session::refreshSnapshot() { + if (editor_) snapshot_ = std::make_shared(editor_->timeline()); +} + +void Session::refreshPaths() { + auto paths = std::make_shared(); + for (const LibraryItem& i : library_) paths->emplace(i.media.id.value(), i.path.toStdString()); + paths_ = std::move(paths); +} + +void Session::onTick() { + if (auto error = audio_.take_error()) setNotice(QString::fromStdString(*error)); + if (auto error = scheduler_.take_error()) { + pause(); + fail(QString::fromStdString(*error)); + return; + } + if (speed_ == 1 && audio_.running() && !audio_.healthy()) { + // The device went away (unplugged, sink removed): restart from what was shown, on a new + // output; the clock re-anchors there. + setNotice(QStringLiteral("Audio device changed")); + startPlayback(); + return; + } + std::int64_t target = 0; + if (speed_ == 1 && audio_.running()) { + // Video follows the audio actually heard: drop frames when behind, hold when ahead. + // The selected frame still has to cross the Qt render loop before it reaches the + // display. Prepare the next frame so that the visible image follows the device clock. + target = audio_.audible_sample() * ticksPerSample() / ticksPerFrame() + 1; + } else { + const double elapsed = static_cast(clock_.nsecsElapsed()) * 1e-9 * frameRate() * speed_; + target = play_from_frame_ + static_cast(std::floor(elapsed)); + } + const bool at_end = speed_ > 0 ? target >= lastFrame() : target <= 0; + target = std::clamp(target, 0, lastFrame()); + if (auto shown = scheduler_.take(target)) { + playhead_ = shown->frame * ticksPerFrame(); + if (preview_ != nullptr) preview_->setFrame(std::move(shown->view)); + emit positionChanged(); + } + if (at_end && frame() == target) pause(); +} + +// ------------------------------------------------------------------- viewer frames (paused) + +void Session::requestFrame() { + if (playing()) return; // the scheduler feeds the viewer while playing + wanted_ = playhead_; + if (!busy_) submitFrame(); +} + +void Session::submitFrame() { + if (!wanted_ || !editor_) return; + const std::int64_t frame = *std::exchange(wanted_, std::nullopt) / ticksPerFrame(); + if (!snapshot_) refreshSnapshot(); + busy_ = true; + const unsigned generation = generation_; + frame_job_ = workers_.submit("viewer-frame", [this, generation, frame, timeline = snapshot_, paths = paths_, + width = canvas_width_, height = canvas_height_, + tpf = ticksPerFrame()](oma::JobContext&) { + auto view = build_viewer_frame(*timeline, *paths, width, height, frame, tpf, frames_); + QMetaObject::invokeMethod(this, [this, generation, view = std::move(view)]() mutable { + busy_ = false; + if (generation != generation_) return; + if (!view) { + fail(message(view.error())); + } else if (preview_ != nullptr && !playing()) { + preview_->setFrame(std::move(*view)); + } + if (wanted_) submitFrame(); + }, Qt::QueuedConnection); + return oma::Result{}; + }); +} + +// ------------------------------------------------------------------- status + +void Session::setNotice(const QString& text) { + notice_ = text; + notice_timer_.start(kNoticeMs); + emit statusChanged(); +} + +void Session::fail(const QString& text) { + failed_ = true; + status_ = text; + emit statusChanged(); +} diff --git a/apps/omamovie/src/session.hpp b/apps/omamovie/src/session.hpp new file mode 100644 index 0000000..695de8a --- /dev/null +++ b/apps/omamovie/src/session.hpp @@ -0,0 +1,237 @@ +#pragma once + +#include "audio_player.hpp" +#include "frame_source.hpp" +#include "video_scheduler.hpp" +#include "viewer_frame.hpp" + +#include "oma/base/jobs.hpp" +#include "oma/base/time.hpp" +#include "oma/media/format.hpp" +#include "oma/timeline/edit.hpp" +#include "oma/timeline/editor.hpp" +#include "oma/timeline/ids.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +class PreviewItem; + +// The bridge between QML and the editing engine (CLAUDE.md §10–§11): the media library, the +// M5 timeline Editor (every edit a command, with undo/redo), the playhead and the viewer. +// QML reads properties and calls intents; it holds no timeline state of its own. +// +// Threading: lives on the UI thread. Probing, thumbnails and viewer frames run on a single +// job worker, which alone touches `frames_`; results come back through queued invocations and +// are dropped when a newer project replaced the one that asked (`generation_`). +// +// Time: the playhead is an integer tick of the sequence timebase, always on the frame grid. +// Seconds cross into QML for display only. +class Session final : public QObject { + Q_OBJECT + Q_PROPERTY(bool editing READ editing NOTIFY viewChanged) + Q_PROPERTY(QString status READ status NOTIFY statusChanged) + Q_PROPERTY(bool failed READ failed NOTIFY statusChanged) + Q_PROPERTY(QString notice READ notice NOTIFY statusChanged) + Q_PROPERTY(QVariantList media READ media NOTIFY libraryChanged) + Q_PROPERTY(int selectedMedia READ selectedMedia NOTIFY selectionChanged) + Q_PROPERTY(QVariantList clips READ clips NOTIFY sequenceChanged) + Q_PROPERTY(double selectedClip READ selectedClip NOTIFY selectionChanged) + Q_PROPERTY(QVariantMap info READ info NOTIFY selectionChanged) + Q_PROPERTY(bool hasMedia READ hasMedia NOTIFY sequenceChanged) + Q_PROPERTY(double duration READ duration NOTIFY sequenceChanged) + Q_PROPERTY(double frameRate READ frameRate NOTIFY sequenceChanged) + Q_PROPERTY(bool canUndo READ canUndo NOTIFY sequenceChanged) + Q_PROPERTY(bool canRedo READ canRedo NOTIFY sequenceChanged) + Q_PROPERTY(QString undoText READ undoText NOTIFY sequenceChanged) + Q_PROPERTY(QString redoText READ redoText NOTIFY sequenceChanged) + Q_PROPERTY(double position READ position NOTIFY positionChanged) + Q_PROPERTY(bool playing READ playing NOTIFY positionChanged) + Q_PROPERTY(int speed READ speed NOTIFY positionChanged) + +public: + explicit Session(QObject* parent = nullptr); + explicit Session(std::unique_ptr importer, QObject* parent = nullptr); + ~Session() override; + Session(const Session&) = delete; + Session& operator=(const Session&) = delete; + Session(Session&&) = delete; + Session& operator=(Session&&) = delete; + + void setPreview(PreviewItem* preview) { preview_ = preview; } + // Automated runs: never play through the speakers. + void setSilent(bool silent) { audio_.set_silent(silent); } + void setOutputMuted(bool muted) { audio_.set_muted(muted); } + [[nodiscard]] float takeAudioPeak() { return audio_.take_peak(); } + [[nodiscard]] bool audioRunning() const { return audio_.running(); } + [[nodiscard]] bool audioOnDevice() const { return audio_.on_device(); } + [[nodiscard]] std::int64_t audibleSample() const { return audio_.audible_sample(); } + [[nodiscard]] std::int64_t audioUnderruns() const { return audio_.underruns(); } + [[nodiscard]] std::int64_t droppedVideoFrames() const { return scheduler_.dropped(); } + + [[nodiscard]] bool editing() const { return editing_; } + [[nodiscard]] QString status() const { return status_; } + [[nodiscard]] bool failed() const { return failed_; } + [[nodiscard]] QString notice() const { return notice_; } + [[nodiscard]] QVariantList media() const; + [[nodiscard]] int selectedMedia() const { return selected_media_; } + [[nodiscard]] QVariantList clips() const; + [[nodiscard]] double selectedClip() const { return static_cast(selected_clip_.value()); } + [[nodiscard]] QVariantMap info() const; + [[nodiscard]] bool hasMedia() const; + [[nodiscard]] double duration() const; + [[nodiscard]] double frameRate() const; + [[nodiscard]] bool canUndo() const { return editor_ && editor_->can_undo(); } + [[nodiscard]] bool canRedo() const { return editor_ && editor_->can_redo(); } + [[nodiscard]] QString undoText() const; + [[nodiscard]] QString redoText() const; + [[nodiscard]] double position() const; + [[nodiscard]] bool playing() const { return speed_ != 0; } + // Playback speed: 0 paused, 1 normal, 2 and 4 faster, negative in reverse (J/K/L). + [[nodiscard]] int speed() const { return speed_; } + [[nodiscard]] std::int64_t frame() const; + + // Screens (ui-design §3). + Q_INVOKABLE void newProject(); + Q_INVOKABLE void showProjects(); + Q_INVOKABLE void continueProject(); + + // Import. A file opened with the app goes straight to the end of the storyline (§3.1); one + // imported from the dialog goes to the library, and to the storyline only when it is empty. + Q_INVOKABLE void open(const QString& path) { importFile(path, true); } + Q_INVOKABLE void importUrl(const QUrl& url); + + Q_INVOKABLE void selectMedia(int index); + Q_INVOKABLE void selectClip(double id); + + // Edits (ui-design §8.1). Each is one command in the history. + Q_INVOKABLE void appendSelected(); + Q_INVOKABLE void insertSelected(); + Q_INVOKABLE void overwriteSelected(); + Q_INVOKABLE void splitAtPlayhead(); + Q_INVOKABLE void deleteSelected(bool ripple); + // Drags an edge by whole frames; the storyline stays magnetic (ripple, ui-design §7.3). + Q_INVOKABLE void trimClip(double id, bool head, int frames); + // Volume drawer (ui-design §6): one command per committed change. Fades are snapped to + // whole frames and clamped so both fit in the clip. + Q_INVOKABLE void setClipAudio(double gain, double fadeIn, double fadeOut, bool muted); + Q_INVOKABLE void undo(); + Q_INVOKABLE void redo(); + + // Transport. + Q_INVOKABLE void togglePlay(); + Q_INVOKABLE void pause(); + // J/K/L (ui-design §8.1): +1 is L (play forward, faster on repeat), -1 is J (reverse, + // faster on repeat); K is pause(). Audio plays only at normal speed; at other speeds the + // clock is monotonic. + Q_INVOKABLE void shuttle(int direction); + Q_INVOKABLE void seek(double seconds); + Q_INVOKABLE void stepFrames(int frames); + Q_INVOKABLE void toEnd(); + +signals: + void viewChanged(); + void statusChanged(); + void libraryChanged(); + void selectionChanged(); + void sequenceChanged(); + void positionChanged(); + +private: + struct LibraryItem { + oma::timeline::MediaInfo media; + QString path; + QString name; + QString thumbnail; + double seconds = 0; // display only + std::optional rate; + std::uint32_t width = 0; // display size, rotation applied + std::uint32_t height = 0; + QVariantMap details; // codec, resolution, frame rate for the Info drawer + }; + + void importFile(const QString& path, bool append); + void addToLibrary(LibraryItem item, bool append); + [[nodiscard]] const LibraryItem* item(oma::timeline::MediaId id) const; + bool ensureSequence(const LibraryItem& first); + [[nodiscard]] std::optional sourceFor(const LibraryItem& item) const; + bool run(std::unique_ptr command); + void afterEdit(); + void placeSelected(int how); + void setNotice(const QString& text); + void fail(const QString& message); + + [[nodiscard]] std::int64_t ticksPerFrame() const; + [[nodiscard]] std::int64_t lastFrame() const; + void setFrame(std::int64_t frame); + void startPlayback(); + void setSpeed(int speed); + void refreshSnapshot(); + void refreshPaths(); + [[nodiscard]] std::int64_t ticksPerSample() const; + void onTick(); + void requestFrame(); + void submitFrame(); + + PreviewItem* preview_ = nullptr; + bool editing_ = false; + QString status_; + bool failed_ = false; + QString notice_; + QTimer notice_timer_; + + std::vector library_; + int selected_media_ = -1; + std::uint64_t next_media_ = 1; + + std::optional editor_; + oma::timeline::TrackId primary_; // the storyline (ui-design §7.1) + // The frame size of the sequence, from its first clip (ui-design §3: format from the first + // clip; the aspect-ratio choice of ADR-0010 comes later). + std::uint32_t canvas_width_ = 1920; + std::uint32_t canvas_height_ = 1080; + oma::timeline::ClipId selected_clip_; + + std::int64_t playhead_ = 0; // sequence ticks, on the frame grid + int speed_ = 0; + // The master clock is the audio actually played (CLAUDE.md §12): video follows it and + // drops frames it cannot decode in time. Only if audio cannot start at all does a monotonic + // clock take over. + QElapsedTimer clock_; + std::int64_t play_from_frame_ = 0; + QTimer tick_; + VideoScheduler scheduler_; + // Immutable views shared with the pipelines: the timeline as of the last edit and the + // library's files. Replaced, never mutated. + std::shared_ptr snapshot_; + std::shared_ptr paths_ = std::make_shared(); + + // Viewer frame requests: one in flight, and only the latest wish kept. + bool busy_ = false; + std::optional wanted_; + unsigned generation_ = 0; + + // Imports in flight (cancelled by New project) and the viewer frame being decoded. + std::vector imports_; + oma::JobHandle frame_job_; + + QTemporaryDir thumbnails_; + // Accessed only by the import job worker; its lifetime extends past worker shutdown. + std::unique_ptr importer_; + FrameSource frames_; // job worker only + AudioPlayer audio_; + bool warned_silent_ = false; + oma::JobPool workers_{1}; // destroyed first: no job outlives what it touches +}; diff --git a/apps/omamovie/src/timeline_audio.cpp b/apps/omamovie/src/timeline_audio.cpp new file mode 100644 index 0000000..93f347b --- /dev/null +++ b/apps/omamovie/src/timeline_audio.cpp @@ -0,0 +1,164 @@ +#include "timeline_audio.hpp" + +#include "oma/audio/mix.hpp" + +#include +#include +#include + +namespace tl = oma::timeline; + +namespace { + +__extension__ using Int128 = __int128; + +Int128 floor_div(Int128 a, Int128 b) { + const Int128 q = a / b; + return (a % b != 0 && ((a < 0) != (b < 0))) ? q - 1 : q; +} + +std::int64_t ceil_div(std::int64_t a, std::int64_t b) { + return -static_cast(floor_div(-static_cast(a), b)); +} + +// floor((t + ticks * unit) * rate): the media sample at a time offset, exact (128-bit). +std::int64_t sample_at(const oma::RationalTime& t, std::int64_t ticks, oma::Rational unit, std::int32_t rate) { + const oma::Rational tb = t.timebase(); + const Int128 den = static_cast(tb.den()) * unit.den(); + const Int128 num = (static_cast(t.value()) * tb.num() * unit.den() + + static_cast(ticks) * unit.num() * tb.den()) * + rate; + return static_cast(floor_div(num, den)); +} + +} // namespace + +TimelineAudio::TimelineAudio(tl::Timeline timeline, std::unordered_map paths, + oma::SampleRate rate, int channels) + : timeline_(std::move(timeline)), paths_(std::move(paths)), rate_(rate), channels_(channels) { + // The sequence timebase holds whole samples (Timeline::default_timebase). + ticks_per_sample_ = std::max(1, timeline_.to_ticks(rate_.sample_to_time(1)).value_or(1)); +} + +oma::Result TimelineAudio::render(std::span out, std::int64_t first) { + ++pass_; + std::ranges::fill(out, 0.0F); + const auto frames = static_cast(out.size() / static_cast(channels_)); + oma::Result result; + for (const tl::Track& track : timeline_.tracks()) { + if (track.muted || track.kind == tl::TrackKind::Caption) { + continue; + } + for (const tl::Clip& clip : track.clips) { + const tl::MediaInfo* media = timeline_.find_media(clip.media); + if (media == nullptr || !media->has_audio || clip.audio.muted) { + continue; + } + // The clip owns the samples whose start lies inside [start, end). + const std::int64_t a = ceil_div(clip.start_ticks(), ticks_per_sample_); + const std::int64_t b = ceil_div(clip.end_ticks(), ticks_per_sample_); + if (b <= first || a >= first + frames) { + continue; + } + if (auto r = mix_clip(clip, out, first); !r && result) { + result = r; // report the first failure; the rest of the mix still plays + } + } + } + std::erase_if(streams_, [&](const auto& entry) { return entry.second.used_in != pass_; }); + return result; +} + +oma::Result TimelineAudio::stream(const tl::Clip& clip) { + auto [it, created] = streams_.try_emplace(clip.id.value()); + Stream& s = it->second; + s.used_in = pass_; + if (!created) { + return &s; + } + s.next_sample = -1; // forces the first seek + const auto path = paths_.find(clip.media.value()); + if (path == paths_.end()) { + s.failed = true; + return oma::make_error(oma::ErrorCode::InvalidArgument, oma::Category::Audio, "clip media has no file"); + } + oma::media::AudioDecoderOptions options; + options.sample_rate = rate_; + options.channels = channels_; + auto decoder = oma::media::AudioDecoder::open(std::filesystem::path(path->second), options); + if (!decoder) { + s.failed = true; + return std::unexpected(decoder.error()); + } + s.decoder = std::move(*decoder); + return &s; +} + +oma::Result TimelineAudio::mix_clip(const tl::Clip& clip, std::span out, std::int64_t first) { + if (clip.time_map.speed() != oma::Rational::literal(1, 1)) { + return {}; // needs time-stretching (v0.2 speed work) + } + auto opened = stream(clip); + if (!opened) { + return std::unexpected(opened.error()); + } + Stream& s = **opened; + if (s.failed) { + return {}; + } + const auto frames = static_cast(out.size() / static_cast(channels_)); + const std::int64_t a = ceil_div(clip.start_ticks(), ticks_per_sample_); + const std::int64_t b = ceil_div(clip.end_ticks(), ticks_per_sample_); + const std::int64_t from = std::max(a, first); + const std::int64_t to = std::min(b, first + frames); + const std::int64_t media_sample = + sample_at(clip.source_in, (from * ticks_per_sample_) - clip.start_ticks(), timeline_.timebase(), rate_.hz()); + if (s.next_sample != media_sample) { + // A jump (first block, a seek, or a cut back into this media): reposition exactly. + auto t = oma::RationalTime::make(media_sample, rate_.timebase()); + if (!t) { + return std::unexpected(t.error()); + } + if (auto r = s.decoder->seek(*t); !r) { + s.failed = true; + return r; + } + s.buffer.reset(); + s.offset = 0; + s.ended = false; + s.next_sample = media_sample; + } + const oma::audio::ClipGain gain{ + .gain = clip.audio.gain, + .length = b - a, + .fade_in = rate_.time_to_sample(clip.audio.fade_in, oma::Rounding::Floor).value_or(0), + .fade_out = rate_.time_to_sample(clip.audio.fade_out, oma::Rounding::Floor).value_or(0)}; + std::int64_t pos = from; + while (pos < to) { + if (!s.buffer || s.offset >= s.buffer->frames) { + if (s.ended) { + break; + } + auto next = s.decoder->next(); + if (!next) { + s.failed = true; + return std::unexpected(next.error()); + } + if (!*next) { + s.ended = true; + break; + } + s.buffer = std::move(*next); + s.offset = 0; + } + const std::int64_t n = std::min(to - pos, s.buffer->frames - s.offset); + const auto at = static_cast((pos - first) * channels_); + oma::audio::mix_planar(out.subspan(at), channels_, + std::span(s.buffer->samples).subspan(static_cast(s.offset)), + s.buffer->channels, s.buffer->frames, n, gain, pos - a); + s.offset += n; + s.next_sample += n; + pos += n; + } + return {}; +} diff --git a/apps/omamovie/src/timeline_audio.hpp b/apps/omamovie/src/timeline_audio.hpp new file mode 100644 index 0000000..3a0af37 --- /dev/null +++ b/apps/omamovie/src/timeline_audio.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include "oma/base/error.hpp" +#include "oma/base/time.hpp" +#include "oma/media/audio_decoder.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include +#include + +// Renders the sequence's audio, sample-accurate, at the output rate (CLAUDE.md §6, §12): for each +// block, every audible clip's media is decoded at the right position and mixed with its gain and +// fades. Works on a snapshot of the timeline taken when playback starts. +// +// Threading: owned by the playback audio pipeline thread; never the real-time callback (it +// decodes, allocates and does I/O). +class TimelineAudio { +public: + TimelineAudio(oma::timeline::Timeline timeline, std::unordered_map paths, + oma::SampleRate rate, int channels); + + // Fills `out` (interleaved) with sequence samples [first, first + out.size() / channels). + // Clips at speeds other than 1 stay silent until time-stretching exists. Media that cannot + // be decoded is silent and reported once through the result. + [[nodiscard]] oma::Result render(std::span out, std::int64_t first); + + [[nodiscard]] int channels() const noexcept { return channels_; } + +private: + struct Stream { + std::unique_ptr decoder; + std::optional buffer; + std::int64_t offset = 0; // frames of `buffer` already used + std::int64_t next_sample = 0; // media sample at `buffer[offset]` + bool ended = false; + bool failed = false; + std::uint64_t used_in = 0; // last render pass that touched it + }; + + [[nodiscard]] oma::Result mix_clip(const oma::timeline::Clip& clip, std::span out, + std::int64_t first); + [[nodiscard]] oma::Result stream(const oma::timeline::Clip& clip); + + oma::timeline::Timeline timeline_; + std::unordered_map paths_; // media ID -> file + oma::SampleRate rate_; + int channels_; + std::int64_t ticks_per_sample_ = 1; + std::unordered_map streams_; // per clip ID + std::uint64_t pass_ = 0; +}; diff --git a/apps/omamovie/src/video_scheduler.cpp b/apps/omamovie/src/video_scheduler.cpp new file mode 100644 index 0000000..6586b8c --- /dev/null +++ b/apps/omamovie/src/video_scheduler.cpp @@ -0,0 +1,78 @@ +#include "video_scheduler.hpp" + +#include + +VideoScheduler::~VideoScheduler() { + stop(); + pipeline_.shutdown(); +} + +void VideoScheduler::start(std::shared_ptr timeline, + std::shared_ptr paths, std::uint32_t width, std::uint32_t height, + std::int64_t ticks_per_frame, std::int64_t from, int step, std::int64_t last) { + stop(); + step_ = step == 0 ? 1 : step; + target_.store(from, std::memory_order_relaxed); + auto queue = std::make_shared>(kAhead); + queue_ = queue; + const int stride = step_; + producer_ = pipeline_.submit("playback-video", [this, queue, timeline = std::move(timeline), + paths = std::move(paths), width, height, ticks_per_frame, + from, stride, last](oma::JobContext& ctx) { + FrameSource frames; // this pipeline's own decoders + std::int64_t next = from; + while (!ctx.is_cancelled()) { + // Never prepare a frame the clock has already passed. + const std::int64_t clock = target_.load(std::memory_order_relaxed); + if (stride > 0 ? next < clock : next > clock) { + next = clock; + } + if (next < 0 || next > last) { + break; // the end in this direction; the clock's owner stops playback + } + auto view = build_viewer_frame(*timeline, *paths, width, height, next, ticks_per_frame, frames); + if (!view) { + const std::scoped_lock lock(error_mutex_); + error_ = view.error().summary(); + break; + } + if (!queue->push(Frame{.frame = next, .view = std::move(*view)})) { + break; // closed: stopped + } + next += stride; + } + return oma::Result{}; + }); +} + +void VideoScheduler::stop() noexcept { + if (queue_) { + queue_->close(); // releases a producer blocked on a full queue + } + if (producer_.valid()) { + producer_.cancel(); + (void)producer_.wait(); + producer_ = {}; + } + queue_.reset(); +} + +std::optional VideoScheduler::take(std::int64_t target) { + target_.store(target, std::memory_order_relaxed); + std::optional shown; + if (!queue_) { + return shown; + } + while (auto f = queue_->try_pop_if([&](const Frame& candidate) { return not_after(candidate.frame, target); })) { + if (shown) { + dropped_.fetch_add(1, std::memory_order_relaxed); + } + shown = std::move(f); + } + return shown; +} + +std::optional VideoScheduler::take_error() { + const std::scoped_lock lock(error_mutex_); + return std::exchange(error_, std::nullopt); +} diff --git a/apps/omamovie/src/video_scheduler.hpp b/apps/omamovie/src/video_scheduler.hpp new file mode 100644 index 0000000..fdc4cf1 --- /dev/null +++ b/apps/omamovie/src/video_scheduler.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include "viewer_frame.hpp" + +#include "oma/base/bounded_queue.hpp" +#include "oma/base/jobs.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include +#include + +// Video side of playback (M4 playback scheduler): a pipeline thread decodes and prepares the +// frames ahead of the clock into a bounded queue; the clock's owner takes, at each tick, the +// newest frame not past the clock and drops the older ones. Video follows the master clock by +// dropping or repeating frames (CLAUDE.md §12). +// +// Threading: start/stop/take run on the UI thread. The producer runs on this scheduler's own +// one-thread JobPool (the playback pipeline of ADR-0003) with its own decoders, works on an +// immutable timeline snapshot, and blocks when the queue is full (backpressure). stop() closes +// the queue and joins the producer, so cancellation on seek or speed change is immediate. +class VideoScheduler { +public: + struct Frame { + std::int64_t frame = 0; + std::shared_ptr view; + }; + + // Frames prepared ahead: about a quarter second at 30 fps, bounded memory. + static constexpr std::size_t kAhead = 8; + + VideoScheduler() = default; + ~VideoScheduler(); + VideoScheduler(const VideoScheduler&) = delete; + VideoScheduler& operator=(const VideoScheduler&) = delete; + VideoScheduler(VideoScheduler&&) = delete; + VideoScheduler& operator=(VideoScheduler&&) = delete; + + // Starts preparing frames from `from`, `step` frames apart (2 for double speed, -1 for + // reverse), up to frame `last` (or 0 in reverse). + void start(std::shared_ptr timeline, std::shared_ptr paths, + std::uint32_t width, std::uint32_t height, std::int64_t ticks_per_frame, std::int64_t from, + int step, std::int64_t last); + void stop() noexcept; + + // The newest prepared frame at or before `target` in the playback direction, if any is new; + // older prepared frames are dropped. Also tells the producer where the clock is, so a late + // producer skips ahead instead of preparing frames nobody will show. + [[nodiscard]] std::optional take(std::int64_t target); + + // Prepared frames never shown because a newer one was due. + [[nodiscard]] std::int64_t dropped() const noexcept { return dropped_.load(std::memory_order_relaxed); } + // A failure in the producer (undecodable media), reported once. + [[nodiscard]] std::optional take_error(); + +private: + [[nodiscard]] bool not_after(std::int64_t frame, std::int64_t target) const noexcept { + return step_ > 0 ? frame <= target : frame >= target; + } + + int step_ = 1; + std::shared_ptr> queue_; + std::atomic target_{0}; + std::atomic dropped_{0}; + std::mutex error_mutex_; + std::optional error_; + oma::JobHandle producer_; + oma::JobPool pipeline_{1}; // destroyed first: the producer never outlives what it uses +}; diff --git a/apps/omamovie/src/viewer_frame.cpp b/apps/omamovie/src/viewer_frame.cpp new file mode 100644 index 0000000..8fb3e82 --- /dev/null +++ b/apps/omamovie/src/viewer_frame.cpp @@ -0,0 +1,55 @@ +#include "viewer_frame.hpp" + +#include "oma/timeline/evaluate.hpp" + +#include + +namespace tl = oma::timeline; + +namespace { + +oma::compositor::Layer to_layer(const tl::VideoProperties& v, std::size_t input) { + oma::compositor::Layer layer; + layer.input = input; + layer.fit = static_cast(v.fit); // same enumerators, same order + layer.crop = {.left = v.crop.left, .top = v.crop.top, .right = v.crop.right, .bottom = v.crop.bottom}; + layer.transform = {.offset_x = v.transform.offset_x, + .offset_y = v.transform.offset_y, + .scale_x = v.transform.scale_x, + .scale_y = v.transform.scale_y, + .rotation = v.transform.rotation}; + layer.opacity = v.opacity; + layer.blend = static_cast(v.blend); + return layer; +} + +} // namespace + +oma::Result> build_viewer_frame(const tl::Timeline& timeline, const MediaPaths& paths, + std::uint32_t width, std::uint32_t height, + std::int64_t frame, std::int64_t ticks_per_frame, + FrameSource& frames) { + const auto at = timeline.at(frame * ticks_per_frame); + auto composition = tl::evaluate(timeline, at); + if (!composition) { + return std::unexpected(composition.error()); + } + auto out = std::make_shared(); + out->graph.width = width; + out->graph.height = height; + out->frame = frame; + out->seconds = at.seconds_approx(); + for (const tl::VideoLayer& layer : composition->video) { + const auto path = paths.find(layer.media.value()); + if (path == paths.end()) { + continue; // media not in the library (cannot happen while the library only grows) + } + auto picture = frames.picture_at(path->second, layer.media_time); + if (!picture) { + return std::unexpected(picture.error()); + } + out->graph.layers.push_back(to_layer(layer.video, out->pictures.size())); + out->pictures.push_back(std::move(*picture)); + } + return out; +} diff --git a/apps/omamovie/src/viewer_frame.hpp b/apps/omamovie/src/viewer_frame.hpp new file mode 100644 index 0000000..fedd782 --- /dev/null +++ b/apps/omamovie/src/viewer_frame.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "frame_source.hpp" + +#include "oma/base/error.hpp" +#include "oma/compositor/render_graph.hpp" +#include "oma/timeline/model.hpp" + +#include +#include +#include +#include +#include + +// Media ID -> file, as the library knows it. +using MediaPaths = std::unordered_map; + +// What the viewer shows at one instant: the render graph built from the timeline and one +// decoded picture per layer input. +struct ViewerFrame { + oma::compositor::RenderGraph graph; + std::vector pictures; + std::int64_t frame = 0; // sequence frame + double seconds = 0; // its time, for display and measurements only +}; + +// Builds the frame at sequence frame `frame`: evaluates the timeline (every visible layer, +// bottom first, with its fit/crop/transform/opacity/blend; a gap is the black background) and +// decodes each layer's picture with `frames`. Runs on the thread that owns `frames`. +[[nodiscard]] oma::Result> +build_viewer_frame(const oma::timeline::Timeline& timeline, const MediaPaths& paths, std::uint32_t width, + std::uint32_t height, std::int64_t frame, std::int64_t ticks_per_frame, FrameSource& frames); From c547eeb3d67d95f0b5638d3da5bacee323fcc406 Mon Sep 17 00:00:00 2001 From: KitsuneSemCalda Date: Sat, 3 Oct 2026 20:38:14 -0300 Subject: [PATCH 011/464] docs(research): revise research notes after a skeptical evidence review --- Docs/Research/README.md | 198 +--- Docs/Research/capcut.md | 164 +-- Docs/Research/cuda.md | 210 ++-- Docs/Research/davinci-resolve.md | 331 +----- .../evidence/2026-10-03-calculations.txt | 11 + Docs/Research/evidence/2026-10-03-local.txt | 48 + .../evidence/2026-10-03-make-test.txt | 960 ++++++++++++++++++ Docs/Research/hardware-strategy.md | 210 ++-- Docs/Research/imovie.md | 255 +---- Docs/Research/movie-maker.md | 185 +--- Docs/Research/omarchy-integration.md | 292 ++---- Docs/Research/opencl.md | 156 +-- Docs/Research/other-editors.md | 200 ++-- Docs/Research/premiere-pro.md | 301 +----- Docs/Research/skeptical-review.md | 178 ++++ Docs/Research/sources.md | 72 ++ Docs/Research/vulkan.md | 335 ++---- 17 files changed, 1947 insertions(+), 2159 deletions(-) create mode 100644 Docs/Research/evidence/2026-10-03-calculations.txt create mode 100644 Docs/Research/evidence/2026-10-03-local.txt create mode 100644 Docs/Research/evidence/2026-10-03-make-test.txt create mode 100644 Docs/Research/skeptical-review.md create mode 100644 Docs/Research/sources.md diff --git a/Docs/Research/README.md b/Docs/Research/README.md index 04e4325..83b0dcf 100644 --- a/Docs/Research/README.md +++ b/Docs/Research/README.md @@ -1,150 +1,62 @@ # Research -Market, UX and technology research that guides OmaMovie. It is not a specification: -recommendations here only become rules once they reach `CLAUDE.md` or an ADR. +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. -**Product and UX** +The architecture remains a defensible direction, subject to driver, synchronization, +color and workload checks. The product positioning and UX proposals remain hypotheses. +The 2026-10-03 review replaces the previous unsourced market rankings and broad hardware +claims; it does not certify the implementation or complete the pending spikes. -| Document | Topic | -|---|---| -| [premiere-pro.md](premiere-pro.md) | Why Adobe Premiere Pro is a powerful editor (engine, editing model, color, interop, weaknesses) | -| [imovie.md](imovie.md) | Apple iMovie UI/UX (layout, contextual inspector, magnetic timeline, templates) | -| [movie-maker.md](movie-maker.md) | Windows Movie Maker and its successors UI/UX (storyboard/timeline, Clipchamp) | -| [davinci-resolve.md](davinci-resolve.md) | Why DaVinci Resolve is powerful (pages, Cut page, node color, database projects, OTIO, the Linux situation) | -| [capcut.md](capcut.md) | What people like and dislike about CapCut (templates, captions, speed ramps, paywall, terms of service) | -| [other-editors.md](other-editors.md) | Final Cut Pro, Descript, Kdenlive, Shotcut and patterns shared across editors | - -**Technology and platform** - -| Document | Topic | -|---|---| -| [hardware-strategy.md](hardware-strategy.md) | **Synthesis**: role of each API, single Vulkan device, runtime selection, proposed `CLAUDE.md` changes, spikes | -| [vulkan.md](vulkan.md) | Vulkan-Hpp, Vulkan Video, VA-API→Vulkan, FFmpeg `hwcontext_vulkan`, Qt Quick on an external device, libplacebo | -| [cuda.md](cuda.md) | NVDEC/NVENC without the toolkit, CUDA↔Vulkan interop, kernels through PTX (clang), hybrid laptops | -| [opencl.md](opencl.md) | The analysis that led to **removing OpenCL** (rusticl, NEO, ROCm, Vulkan interop, the Blender lesson) | -| [omarchy-integration.md](omarchy-integration.md) | Theme, window opacity, screen recordings, Quickshell shell, menu, installed drivers, distribution | - -## Method - -- Sources: official documentation (Adobe Help, Apple Support, Blackmagic, Kdenlive), Wikipedia, - trade press, independent benchmarks (Puget Systems) and review aggregators (G2, Capterra; - treated as a signal of opinion, not as measurement). Links in each document. -- Claims without a specific source are marked **(general knowledge)** or **(general observation)**. -- Each section ends with **Implications for OmaMovie**, linking the finding to `CLAUDE.md` sections. -- Goal: extract principles. Copying visual identity or proprietary behavior is forbidden (`CLAUDE.md` §11). - ---- - -## Synthesis - -### Where each product sits - -``` -simple ◄───────────────────────────────────────────────────────────────► powerful -Movie Maker/Photos iMovie CapCut/Clipchamp Shotcut/Kdenlive FCP Premiere/Resolve -(storyboard) (FCP X engine) (presets, social) (free, MLT) (cockpit/pages) - -OmaMovie: a surface close to iMovie/CapCut, an engine close to Premiere/Resolve. -On Linux, the space between Kdenlive/Shotcut and Resolve is empty. -``` - -### Conclusions - -1. **Simplicity comes from contextual disclosure, not from removing features.** iMovie's - adjustments bar and Clipchamp's properties panel show only what applies to the selection. - Premiere solves it with 16 workspaces. OmaMovie's contextual inspector is the right path. - Also adopt the **visual indicator of which adjustments are active** on a clip (iMovie). - -2. **A redesign that removes capability causes rejection.** iMovie '08 and Windows Live Movie - Maker had to bring features back in the following versions. This confirms "hide, do not - remove" and "grow without rewriting the core". - -3. **A professional engine under a simple UI works.** iMovie has used the Final Cut Pro X - engine since 2013 and exports to it. That is OmaMovie's thesis, with the difference of - being a single app: progressive disclosure needs more care. - -4. **Named operations over a generic compositor.** iMovie's PiP, cutaway, split screen and - green screen are presets. In OmaMovie they must produce normal layers and parameters in the - render graph, editable afterwards, without iMovie's layer ceiling. +## Reading order -5. **Semantic storyboard ↔ timeline zoom** (Movie Maker 2012). A single TimelineView whose - minimum zoom becomes a storyboard (equal-size clips, focus on order). A strong UX - differentiator. Requires virtualization and a thumbnail cache. +1. [Skeptical review](skeptical-review.md): hypotheses, corrections, foundations and gates. +2. [Source register](sources.md): documents actually inspected and their limits. +3. [Hardware strategy](hardware-strategy.md): bounded application to the pipeline. +4. [Implementation plan](../implementation-plan.md): milestone work and acceptance criteria. -6. **"Powerful" starts in the engine**: real-time GPU playback of native media, hardware - decode/encode with a fallback, a float pipeline, proxies. Premiere also shows that - **effects without a GPU path need a visible indication** (the render bar). +## Documents -7. **A complete editing model in the core, a gradual UI.** Ripple/roll/slip/slide, split edits - and three-point editing should exist as timeline commands early. iMovie shows roll and - split edits can have a friendly UI (Precision Editor, Clip Trimmer). Premiere shows full - keyboard access. - -8. **Automatic color management** (Premiere 25.2: color space detection, presets, effects - that know their color space). Feeds ADR-0006. - -9. **Stability is part of power.** The main criticism of Premiere is instability and corrupted - projects. Atomic save, tested migrations and parser fuzzing are a competitive advantage, - not bureaucracy. - -10. **`.prproj` is gzip-compressed XML.** A relatively accessible import target to validate - `Importer → ProjectIR`, after OTIO/FCPXML/EDL. - -11. **Resolve on Linux leaves an opening.** The free version does not decode H.264/H.265 or - AAC on Linux, official support is NVIDIA only and the documented audio path is ALSA. Phone - video + an AMD/Intel GPU + PipeWire, the typical Omarchy setup, is poorly served. This is - OmaMovie's **clearest positioning**. - -12. **Simple and fast are not opposites.** Blackmagic created the Cut page (source tape, dual - timeline) for professionals on a deadline. CapCut wins on time to result. OmaMovie's - simplicity must also serve advanced users. - -13. **Overview + detail in the timeline**: Resolve's dual timeline and Movie Maker's semantic - zoom attack the same problem. Prototype both approaches (or a minimap) in the TimelineView. - -14. **Graph inside, stack outside.** Resolve 21 added a layer-list view to its color node - graph. OmaMovie's render graph is a DAG; the UI shows the stack. - -15. **Projects as files, not in a database.** Resolve projects live in a database - (local/PostgreSQL/cloud) and need a `.drp` export to become a file. This confirms - `CLAUDE.md` §14/§17: the file is the source of truth. - -16. **OTIO is the first interop target.** Resolve has exported OTIO natively since 18.5: a - Resolve → OmaMovie path without reverse engineering. - -17. **Outcome operations, not mechanism operations.** CapCut sells "hero time", not "speed - keyframes". Intent presets first, parameters later. - -18. **Competitors' weak points are OmaMovie's differentiators**: paywall and content terms - (CapCut), subscription (Premiere), crashes with lost work (Premiere, CapCut, Shotcut), slow - preview (Shotcut), learning curve and demanding hardware (Resolve). Offline, no account, - stable and GPU-first. - -### Architecture requirements that emerged - -| Requirement | Origin | Affects | -|---|---|---| -| Fast seek and a multi-density thumbnail cache with aggressive cancellation | Skimming (iMovie), semantic zoom (Movie Maker) | `media`, cache, job system | -| Interactive preview overlays that emit commands | Crop/position/eyedropper in the viewer (iMovie) | PreviewPanel, `timeline` | -| Magnetism as a UI editing policy, not a model restriction | Magnetic timeline (iMovie/FCP X) | `timeline` | -| GPU capability declared per effect and indication of non-real-time sections | Mercury (Premiere) | `compositor`, UI | -| Templates produce a normal, editable timeline | Trailers (iMovie), AutoMovie | `timeline`, `project-ir` | -| Per-stream color metadata and color-space-aware effects | Color management (Premiere) | `media`, `compositor` | -| Editing during playback (dynamic trim) | Premiere | Threading (ADR-0003) | -| **Curve-based time mapping** (not only constant speed), decided before implementing speed | Speed ramps (CapCut) | `base` time, `timeline` (ADR-0002) | -| A first-class caption/text track, SRT/VTT import/export | CapCut, Premiere, Resolve | `timeline`, `project` | -| A canvas with changeable aspect ratio; decide normalized coordinates vs. pixels | Vertical/horizontal (CapCut, Clipchamp) | `compositor`, `timeline` | -| Markers in the timeline model | Beat sync (CapCut), review | `timeline` | -| Serializable commands with no UI dependency (base for future scripting/MCP) | Scripting and MCP (Resolve 21.1) | `timeline` | -| An action system independent of the input device | Speed Editor (Resolve) | UI | -| Background render filling the cache when real time is not possible | Final Cut Pro | `compositor`, cache, job system | -| Transcript as a view of the timeline (words with a `TimeRange`) | Descript, Premiere | `timeline` (future) | -| Benchmarks including Intel/AMD integrated GPUs | Resolve needs strong hardware | `tools/bench` | - -### Suggested next research - -- **OpenTimelineIO**: data model as a reference for ProjectIR (priority, given Resolve's native support). -- **Final Cut Pro in depth**: storylines, roles, FCPXML (interop target). -- **Kdenlive/MLT internals**: MLT architecture, why 10-bit and GPU decode arrived late, project format (importer candidate). -- **Linux hardware pipeline**: current state of VA-API, Vulkan Video and DMA-BUF interop on Mesa/NVIDIA (technical base for `CLAUDE.md` §7.3). -- **Avid Media Composer and Vegas**: editing model and formats (future interop targets). +| Document | Scope | +|---|---| +| [Vulkan](vulkan.md) | API contracts, video capabilities, FFmpeg, Qt and libplacebo | +| [CUDA](cuda.md) | NVIDIA runtime/build distinction, interop and proposed comparison | +| [OpenCL](opencl.md) | Scope exclusion; counterevidence to an impossibility claim | +| [Hardware strategy](hardware-strategy.md) | Device ownership, selection, experiments and fallbacks | +| [Omarchy integration](omarchy-integration.md) | Installed-file observations and version-sensitive adapters | +| [Premiere](premiere-pro.md) | Editing, color and importer hypotheses | +| [iMovie](imovie.md) | Contextual controls and transferable UX proposals | +| [Movie Maker / Clipchamp](movie-maker.md) | Historical limits and current documented interactions | +| [Resolve](davinci-resolve.md) | Versioned codec evidence and limits of competitive inference | +| [CapCut](capcut.md) | Documented captions/curves; unmeasured audience preferences | +| [Other editors](other-editors.md) | Credible baselines and bounded comparisons | + +## Evidence rules + +- **Documented** means a cited passage or local API contract supports the stated scope. +- **Observed locally** describes the installed files or a named experiment, with versions. +- **Hypothesis / inference** names a proposed explanation or deduction, with a disconfirming check. +- **Preference** records a product/engineering choice; competitor behavior cannot prove it. +- **Pending** means the decisive check has not run. A checkbox for implementation is not validation. + +Sources are linked beside claims. Multiple vendor pages from one organization are one +lineage, not independent measurements. Marketing pages establish advertised features only. +Unverified release details, prices, legal interpretations and anecdotal complaint rankings +have been removed from active guidance; git history retains the earlier text. + +## Supported synthesis + +GPU compositing, explicit time/ownership, bounded work and recoverable project persistence +have technical reasons independent of competitor reputation. FFmpeg and Qt expose useful +contracts, but their coexistence must be tested (gate Q1). Intel S1/S2 reports support a +bounded Intel path; AMD/NVIDIA and hybrid support remain unvalidated. + +The fixed layout, contextual drawer, magnetic policy and editable presets are product +choices. Compare them with a conventional track/inspector baseline in M6. A minimap and +semantic storyboard zoom are alternatives to evaluate, not requirements proved by history. + +OTIO is a reasonable first interchange pilot because its schema/library are public. This +provides no promise of effect fidelity or lossless conversion of arbitrary time values. +Keep proprietary import read-only and fixture/version scoped. No evidence here establishes +an empty Linux market, demand for replacing Kdenlive, or professional parity with Resolve. diff --git a/Docs/Research/capcut.md b/Docs/Research/capcut.md index db4d9ad..31da8bb 100644 --- a/Docs/Research/capcut.md +++ b/Docs/Research/capcut.md @@ -1,133 +1,55 @@ -# CapCut: what people like (and dislike) +# CapCut: documented features and untested preferences -> Research done on 2026-10-02. Sourced facts are linked in the [Sources](#sources) section. -> Part of the CapCut sources are promotional material or written by competitors (e.g. VEED, -> Podcastle). Read prices and complaints with that in mind. +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. ---- +## 1. Conclusion -## 1. Summary +Editable timed captions and speed curves are documented feature references. They do not +establish market dominance, universal demand, or the right order of OmaMovie milestones. +Offline/no-account operation remains a project preference, not proof of competitor failure. -CapCut (ByteDance) is the most popular editor among short-form creators. People like it for -**time to result**: +## 2. Inspected product evidence -1. **Templates** where you only swap the clips; -2. editable **automatic captions**; -3. ready-made **"trending" effects** (speed ramps, transitions, stickers); -4. a **clean**, drag-and-drop **interface** you learn in minutes; -5. **cross-platform** with sync (phone → desktop). +[CapCut's guide](https://www.capcut.com/resource/how-to-use-capcut) describes speed curves +and editable presets alongside effect keyframes. [Its caption page](https://www.capcut.com/tools/auto-caption-generator) +describes generating, correcting and styling timed text. These are promotional primary +sources: they establish advertised workflows, not accuracy, completion times or plan +availability for every region/app/version. -And dislike: an **aggressive paywall** (the Pro price doubled in 2025), **terms of service** -that give ByteDance a perpetual license over content (including drafts), **political -instability** (bans) and **crashes that lose work**. +The former price-doubling, bans, perpetual-license/draft claims and billing/crash complaints +were based on commentary, with no verified applicable policy/jurisdiction or representative +sample. They are removed from active guidance. No legal conclusion is made by this audit. -For OmaMovie, CapCut shows **which high-level operations the audience wants**, and its weak -points are obvious differentiators: offline, no account, no paywall and the user's content -under the user's control. +## 3. Foundations: time mapping ---- - -## 2. What people like - -### 2.1 Time to result -The most repeated argument: going from raw footage to a finished video in minutes. Templates, -presets and automation exist for that. - -### 2.2 Templates -- Ready templates where the user **only swaps photos or clips**. -- Templates follow **trends** (TikTok integration), making it easy to join trends and viral challenges. - -### 2.3 Automatic captions -- Generated by AI from the audio and **editable** afterwards. -- Important because a lot of social video is watched **without sound**. -- The feature most often cited as "saves a lot of time". - -### 2.4 Speed ramps (speed curves) -- Speed > Curve with named presets: **bullet, montage, jump cut, hero time, flash in, flash - out**, all customizable. -- They create progressive slow motion and accelerations without manual keyframes. - -### 2.5 Other high-level features -| Feature | What it does | -|---|---| -| Keyframes | On "every setting" | -| Auto Reframe | Detects the subject and recomposes for another aspect ratio (e.g. horizontal → vertical) | -| Beat Sync | Aligns cuts to the music's beat | -| Text-to-speech | Several voices and languages | -| Background removal | Without a green screen | -| Chroma key, stabilization, motion tracking | Free when CapCut became popular | -| Library of stickers, effects, transitions, audio | Ready-made content | - -### 2.6 Interface -- "Clean, straightforward", drag and drop; you start editing within minutes. -- A conventional multitrack timeline on desktop. -- **Mobile-first**: the phone version is the most polished; the desktop one is described as less polished. -- iOS, Android, web and desktop: start on the phone and finish on the computer. - ---- - -## 3. What people dislike - -| Problem | Detail | -|---|---| -| **Price** | In May 2025 Pro went from US$ 9.99 to US$ 19.99/month. Dynamic captions, effects and cloud storage moved behind the paywall | -| **Billing and support** | Complaints about charges after cancellation and slow support, limited to FAQs and email | -| **Terms of service (June 2025)** | A **perpetual, irrevocable, worldwide, royalty-free** license for ByteDance over all uploaded content, **including unpublished drafts**, even after deleting the account. A risk for agencies and NDA-bound content | -| **Availability** | Banned in India since 2020; in the US, banned in January 2025 and, according to VEED's review, unavailable for new downloads in US app stores | -| **Content removal** | May remove content without notice | -| **Stability** | Reports of crashes while editing with **lost work** | - ---- +Speed is the derivative of source position with respect to timeline time. If timeline time +is `t` seconds and dimensionless speed is `s(t)`, source time is +`m(t) = m(0) + integral_0^t s(u) du`. Merely interpolating source timestamps is not always +the same as interpolating speed; define the interpolation and endpoint duration explicitly. +A frozen segment has a duration despite zero source displacement. Reverse playback needs +a decreasing source map and audio policy. Exact endpoints do not make arbitrary curve +samples exact rationals. See the audit calculation and ADR-0002 review note. ## 4. Implications for OmaMovie -### 4.1 Obvious differentiators - -What CapCut does not offer and OmaMovie offers by definition: - -- **Offline and local**: media and project never leave the machine. -- **No account, no terms over the user's content, no feature paywall, no watermark.** -- **Stability and preservation of work**: atomic save and autosave (`CLAUDE.md` §14). Lost work - is the most serious complaint about any editor. - -### 4.2 Features to consider and their architectural impact - -| CapCut feature | Recommendation | Architectural impact | +| Choice | Supported application | Pending dependency | |---|---|---| -| **Speed ramps with presets** | Yes, medium term | **Mapping timeline time → media time cannot be "constant speed" only.** It needs a **curve** (piecewise monotonic function), evaluated in `RationalTime`. Decide in ADR-0002 (time representation) **before** implementing speed, even if the first UI only has constant speed. Frame interpolation (optical flow) is a later, separate step | -| **Keyframes everywhere** | Yes | Already planned. Effect and transform parameters must be animatable by default, not by exception | -| **Captions as a first-class element** | Yes | A **caption/text** track type in the model, with SRT/VTT import/export. Generating captions automatically needs transcription, an AI non-goal; if it ever lands, prefer a **local** (offline) model and keep the result as ordinary editable captions | -| **Changing the project aspect ratio (vertical/horizontal)** | Yes | The project canvas must be resizable without breaking the edit. Decide in the compositor ADR whether transforms are stored in **normalized coordinates** or pixels, and what happens when the aspect ratio changes. Auto Reframe (subject detection) comes later | -| **Templates** | Future | Same conclusion as iMovie: a template produces a normal, editable timeline (`imovie.md` §7) | -| **Beat sync** | Future | Needs beat detection (audio analysis, no heavy AI) and timeline markers. Markers must exist in the model | -| **Named effect/transition presets** | Yes | Presets are data (saved parameters) over generic effects, like iMovie's overlay presets | -| **Text-to-speech, background removal, stock** | Not now | Non-goals (AI/cloud) | -| **Cross-platform sync** | No | Non-goal (cloud). A file-based project already lets users sync with their own tools | - -### 4.3 Audience - -CapCut shows that the vertical short-form audience is large and wants **outcome operations** -("make it dramatic slow motion"), not **mechanism operations** ("add a speed keyframe"). For -OmaMovie: the inspector offers the intent preset first and the parameters afterwards. The same -principle as iMovie's named operations. - -### 4.4 What not to copy - -- A "trends" library and stock content (needs a server). -- A mobile-first interface. -- Account and cloud dependence. - ---- - -## Sources - -- [App Store: CapCut](https://apps.apple.com/app/1500855883) -- [Microsoft Store: CapCut](https://apps.microsoft.com/detail/xp9kn75rrb9nhs) -- [CapCut: How to use CapCut (official guide)](https://capcut.com/resource/how-to-use-capcut) -- [The Nerd in Charge: Why I'm obsessed with CapCut](https://thenerdincharge.substack.com/p/dont-just-survivestorytell-it-why) -- [ABNewswire: CapCut is basically the cheat code for making videos](https://markets.financialcontent.com/lightport.lightport5/article/abnewswire-2025-12-30-capcut-is-basically-the-cheat-code-for-making-videos-that-dont-look-boring) -- [Skillademia: CapCut keyboard shortcuts](https://www.skillademia.com/shortcuts/capcut-shortcuts/) -- [VEED: CapCut review 2026](https://www.veed.io/learn/capcut-review) -- [Podcastle: CapCut's latest terms of service and pricing](https://podcastle.ai/blog/capcut-terms-of-service) -- [Fluxnote: CapCut questions answered (June 2026)](https://fluxnote.io/guides/capcut-questions-answered) -- [Wikipedia (mirror): CapCut](https://wiki.krisyotam.com/content/wikipedia_en_all_nopic_2026-03/CapCut) +| Captions | First-class timed text; SRT/VTT pilot in M8 | Timing/layout/encoding/round-trip tests | +| Speed curves | Explicit TimeMap domain, rounding and interpolation | Complementary ADR and reverse/freeze tests | +| Intent presets | Normal editable commands/parameters | Discoverability/recovery in M6/M8 | +| Resizable canvas | Define transform anchoring before implementation | Coordinate ADR, portrait/landscape tests | +| Transcription | Separate generator of editable timed text | Concrete local use case; remains non-goal initially | + +Imported captions can exist without speech recognition. A preset UI does not require cloud +stock, account infrastructure or a separate project model. + +## 5. Proposed discriminating test + +Compare preset-first with parameter-first controls on identical simple tasks (caption +correction, slow motion, overlay) and mixed-duration media. Record completion/errors, undo, +output timing and ability to adjust after applying a preset. The claim would weaken if +presets hide important changes or slow experienced users without helping beginners. +No task study or caption-accuracy benchmark was executed. Defer high-cost automation until +basic editing proves useful; no feature inherits priority merely from CapCut advertising it. diff --git a/Docs/Research/cuda.md b/Docs/Research/cuda.md index d7d2919..d20c131 100644 --- a/Docs/Research/cuda.md +++ b/Docs/Research/cuda.md @@ -1,149 +1,61 @@ -# CUDA and the NVIDIA path in OmaMovie - -> Research done on 2026-10-02. Sourced facts are linked in the [Sources](#sources) section. -> Items marked **(verify)** need confirmation. -> -> Verified locally: `nvidia-utils` 610.57 provides `libcuda.so`, `libnvcuvid.so` (NVDEC) and -> `libnvidia-encode.so` (NVENC). The system FFmpeg was built with -> `--enable-nvdec --enable-nvenc --enable-cuda-llvm`. Arch's `cuda` package (toolkit) is -> **4.71 GiB** and is not installed. - ---- - -## 1. Summary - -"Using CUDA" in OmaMovie means **three different things**, with very different costs: - -| What | Needs the CUDA toolkit? | Gain | Recommendation | -|---|---|---|---| -| **NVDEC/NVENC** (hardware decode/encode) | **No**. FFmpeg loads the driver libraries at runtime | High: NVENC is about 5x faster than Vulkan Video encode on NVIDIA | **Yes, early** | -| **CUDA ↔ Vulkan interop** (bringing NVDEC frames to the Vulkan compositor) | No, it uses the driver API (`libcuda`) | Needed to keep frames in VRAM | **Yes**, together with NVDEC | -| **Our own CUDA kernels** (effects) | Only at build time (headers/libdevice); at runtime the driver is enough | Depends on the effect; Vulkan Compute covers most | **Only with a measured gain** | - -Conclusion: on NVIDIA the biggest gain comes from NVDEC/NVENC, which **requires neither -distributing nor building with the toolkit**. CUDA kernels are a later specialization. - ---- - -## 2. NVDEC and NVENC - -- Part of the NVIDIA Video Codec SDK. FFmpeg accesses them through `ffnvcodec-headers`, which - **load `libcuda`/`libnvcuvid`/`libnvidia-encode` dynamically**. There is no build-time link - against NVIDIA libraries **(verify: known ffnvcodec behavior)**. -- A frame decoded by NVDEC lives in **CUDA memory** (FFmpeg's `cuda` hwaccel). -- NVENC accepts CUDA frames directly. -- Comparison reported on the NVIDIA forum: Vulkan Video encode ~194 fps vs. NVENC ~910 fps. - -### Omarchy and NVIDIA -Omarchy installs `nvidia-open-dkms`, `nvidia-utils` and `libva-nvidia-driver` (GPUs with GSP) -and sets `LIBVA_DRIVER_NAME=nvidia` and `NVD_BACKEND=direct`. So VA-API exists on NVIDIA, but -through a translation layer on top of NVDEC. - -**For OmaMovie:** on NVIDIA, use NVDEC/NVENC directly (FFmpeg's `cuda` hwaccel), not the VA-API -layer, which adds a translation step and has known DMA-BUF/NV12 limitations when imported by -Vulkan (see `vulkan.md` §4). - ---- - -## 3. CUDA ↔ Vulkan interop - -The compositor is Vulkan, but the NVDEC frame is in CUDA memory. To avoid going through RAM: - -1. OmaMovie creates an **exportable** Vulkan image/buffer (`VK_KHR_external_memory_fd`, opaque fd handle). -2. CUDA imports that memory: `cuImportExternalMemory` (driver API) or `cudaImportExternalMemory` (runtime API). -3. The NVDEC surface is copied into it with a **device-to-device copy** (it stays in VRAM; not - zero-copy, but no download). -4. Synchronization through a **timeline semaphore** exported from Vulkan and imported into CUDA - (`cudaImportExternalSemaphore`, `cudaSignalExternalSemaphoresAsync`, `cudaWaitExternalSemaphoresAsync`). - -**Important restriction:** in CUDA it is **illegal to wait before the corresponding signal has -been issued**. Vulkan timeline semaphores' wait-before-signal does not hold on the CUDA side. -OmaMovie's scheduler must guarantee the order (signal submitted before the wait is submitted in CUDA). - -Alternatives to evaluate: -- **FFmpeg frame mapping** between the `cuda` and `vulkan` contexts **(verify whether FFmpeg 9 - supports `av_hwframe_map`/transfer CUDA→Vulkan without the CPU)**. If it does, it avoids - writing the interop by hand. -- **Vulkan Video decode directly on NVIDIA**: removes the interop. Decode performance (not - encode) must be measured before ruling it out. - ---- - -## 4. Our own CUDA kernels - -### When it would be worth it -- Heavy effects where CUDA has a **measured** advantage over Vulkan Compute on the same GPU - (e.g. mature libraries such as NPP, or kernels relying on specific features). -- `CLAUDE.md` already says: CUDA only where there is a real gain, and effects do not depend - directly on CUDA. - -### Building without `nvcc` and without requiring the toolkit at runtime -The system FFmpeg uses `--enable-cuda-llvm`: its CUDA kernels are compiled by **clang to PTX**. -The same pattern works for OmaMovie: - -1. `.cu` kernels compiled with **clang** to PTX (LLVM has supported CUDA since 3.9). The build - needs the CUDA headers/libdevice (`--cuda-path`). -2. The PTX is **embedded in the binary**. -3. At runtime, `libcuda.so` is loaded with `dlopen` (it ships with `nvidia-utils`) and the PTX - through the driver API (`cuModuleLoadData` / `cuLinkAddData`). The driver JITs it for the GPU present. - -Consequences: -- End users **do not** need to install the toolkit (4.7 GiB). -- Machines without NVIDIA load nothing; the CUDA backend simply does not appear. -- The CUDA build is **optional** (a Make option), so not every developer needs the toolkit. - ---- - -## 5. Hybrid laptops (Intel/AMD + NVIDIA) - -Omarchy detects hybrid GPUs and avoids waking the dGPU needlessly (its detectors read sysfs -instead of `lspci`, because waking the GPU exceeds Hyprland's reload time budget). - -**Implications for OmaMovie:** -- Device choice is a first-class decision: decoding on the dGPU and compositing/presenting on - the iGPU implies **a copy between GPUs** over the bus. -- Proposed initial rule: **one primary device for the whole pipeline** (decode, compositing, - preview), chosen by the user or by a heuristic (dGPU when plugged in?), and logged. -- Do not wake the dGPU just to enumerate capabilities; use sysfs/Vulkan information without - creating a device when possible **(verify)**. - ---- - -## 6. Licensing - -- Loading `libcuda`/`libnvcuvid`/`libnvidia-encode` dynamically from the installed driver is the - model used by FFmpeg and other free projects. -- The CUDA toolkit has its own EULA. Do not redistribute parts of the toolkit; the PTX OmaMovie - generates is project code **(verify the EULA regarding libdevice embedded in the PTX)**. - ---- - -## 7. Recommendations - -1. **Decode/encode phase (M1–M7)**: NVDEC/NVENC through FFmpeg on NVIDIA, no toolkit. -2. **Interop**: try FFmpeg's frame mapping first; if there is no CPU-free path, implement - external memory + timeline semaphore interop in `libs/gpu`. -3. **Benchmark**: NVDEC + interop vs. direct Vulkan Video decode on NVIDIA. Choose by measurement. -4. **Compute `CudaBackend`**: only after the Vulkan `ComputeBackend` exists and an effect has a - measured gain. PTX through clang, `dlopen` at runtime, optional build. -5. **Test hardware**: the current machine has no NVIDIA GPU (`nvidia-utils` is installed, but - `lspci` only shows the Iris Xe). Validating the NVIDIA path needs another machine. - ---- - -## Sources - -- [NVIDIA: CUDA Runtime API — External Resource Interoperability](https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EXTRES__INTEROP.html) -- [NVIDIA: CUDA Driver API — Module Management](https://docs.nvidia.com/cuda/archive/13.2.2/cuda-driver-api/group__CUDA__MODULE.html) -- [NVIDIA Forums: cudaWaitExternalSemaphoresAsync blocks CPU kernel launch](https://forums.developer.nvidia.com/t/cudawaitexternalsemaphoresasync-blocks-cpu-kernel-launch/329028) -- [IWOCL 2025: SYCL interoperability (external memory/semaphores)](https://www.iwocl.org/wp-content/uploads/iwocl-2025-duncan-brawley-sycl-interoperability.pdf) -- [Intel: DPC++ Compatibility Tool — migration examples (Vulkan interop)](https://www.intel.com/content/www/us/en/docs/dpcpp-compatibility-tool/developer-guide-reference/2025-2/migration-examples.html) -- [vulkane: external memory export example](https://docs.rs/crate/vulkane/0.10.1/source/examples/external_memory_export.rs) -- [LLVM: Compiling CUDA with clang](https://llvm.org/docs/_sources/CompileCudaWithLLVM.md.txt) -- [NVIDIA Forums: Building CUDA code with clang](https://forums.developer.nvidia.com/t/building-cuda-code-with-clang/28781) -- [NVIDIA: Video Codec SDK](https://developer.nvidia.com/video-codec-sdk/download) -- [Phoronix: Using NVIDIA's NVENC on Linux with FFmpeg](https://www.phoronix.com/news/MTg0NTY) -- [Phoronix: FFmpeg NVDEC-accelerated H.264 decoding](https://www.phoronix.com/news/FFmpeg-NVDEC-H264-Acceleration) -- [Phoronix: NVIDIA VA-API driver 0.0.18](https://phoronix.com/news/NVIDIA-VA-API-Driver-0.0.18) -- [NVIDIA Forums: Vulkan Video is 5x slower than NVENC](https://forums.developer.nvidia.com/t/vulkan-video-is-5x-slower-than-nvenc/373916) -- Local Omarchy 4.0.4 files: `/usr/share/omarchy/install/hardware/nvidia.sh`, `/usr/share/omarchy/default/hypr/nvidia.lua` +# CUDA and NVIDIA: runtime, interop and comparison + +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. + +## 1. Conclusion + +NVDEC/NVENC are candidates through the already packaged FFmpeg. CUDA effects remain optional +and require measured advantage. No NVIDIA GPU result exists in this review; S7 gates support. +The former approximately 5× encode claim is withdrawn as a selection rule: a forum report +cannot rank different settings, quality, hardware and driver versions for this project. + +## 2. Runtime versus build + +[NVIDIA's FFmpeg guide](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/ffmpeg-with-nvidia-gpu/index.html) +distinguishes runtime use from compiling its suggested toolkit/NPP-enabled configuration. +Its binary can run without the toolkit. For OmaMovie, linking the system FFmpeg is different +from rebuilding FFmpeg or writing CUDA kernels. Check the actual FFmpeg configuration and +SDK/driver compatibility; installed libraries alone do not establish supported devices/codecs. +[FFmpeg nv-codec-headers](https://github.com/FFmpeg/nv-codec-headers) is the interface source, +not proof that every FFmpeg configuration builds without any toolkit components. + +## 3. CUDA ↔ Vulkan + +[NVIDIA's external-resource API](https://docs.nvidia.com/cuda/cuda-runtime-api/cuda_runtime_api/group__CUDART__EXTRES__INTEROP.html) +documents handle-specific memory mappings and semaphore semantics. Binary and timeline +semaphores have different behavior; the old blanket prohibition on CUDA wait-before-signal +was not established by the inspected reference. Specify the exact handle type/API/version +and prove scheduler liveness rather than applying a forum observation universally. + +Possible path: export Vulkan storage, import in CUDA, copy NVDEC output device-to-device, +then synchronize Vulkan consumption. That GPU copy is not allocation-level zero-copy. +Matching the physical GPU, pitch/planes, allocation flags, handle ownership and teardown +are prerequisites. FFmpeg CUDA↔Vulkan mapping support must be checked in the installed +source/API and exercised; it is not assumed by this document. + +## 4. Kernel build + +PTX loaded by the driver and a Clang build are possible future approaches, not a mandated +build architecture. Compiler/toolkit/libdevice compatibility, PTX target, initialization +cost and distribution terms need a concrete kernel/configuration review before adoption. +Do not make toolkit size a permanent architectural premise. Do not copy a vendor guide's +`--enable-nonfree` configuration into releases without the packaging license check. + +## 5. S7: proposed discriminating check + +Compare NVDEC + Vulkan interop, native Vulkan decode and software + upload on one NVIDIA +GPU. Use identical media, verify frame format/content, count host and device copies, and +report seek latency, end-to-end frame times, CPU/GPU/memory and initialization costs. +For encode compare codec/profile, quality or bitrate, rate control, preset, GOP and output +correctness as well as throughput. A faster encode at lower quality is not an equivalent win. +Reject a path on corruption, synchronization errors, unbounded memory or hidden fallback. + +## 6. Hybrid systems and scope + +One selected GPU per pipeline is a preference that limits cross-device complexity. Verify +CUDA UUID against Vulkan identity and the decode render node. Presentation can still involve +the desktop compositor's other GPU; one application device does not remove that cost. +Power-safe enumeration is unvalidated: capability queries may load drivers or wake devices. +Record power behavior before claiming otherwise. No NVIDIA/hybrid benchmark was run. diff --git a/Docs/Research/davinci-resolve.md b/Docs/Research/davinci-resolve.md index dc20bdb..7c28f19 100644 --- a/Docs/Research/davinci-resolve.md +++ b/Docs/Research/davinci-resolve.md @@ -1,303 +1,64 @@ -# DaVinci Resolve: why it is a powerful editor +# Resolve: versioned Linux constraints and product hypotheses -> Research done on 2026-10-02 (current version: Resolve 21.x). Sourced facts are linked in the -> [Sources](#sources) section. Items marked **(general knowledge)** are widely documented but -> have no specific source in this research; verify before using them as requirements. +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. ---- +## 1. Conclusion -## 1. Summary +The inspected codec matrix supports a bounded Linux codec-accessibility opportunity. +It does not establish that only NVIDIA can run Resolve, that ALSA prevents use on PipeWire, +or that the market between consumer and professional Linux editors is empty. -Resolve is powerful for five reasons: +## 2. Version and source boundaries -1. **Cinema-grade color.** It was born as a color correction system, and its node-based grading - is considered the industry standard. -2. **Everything in one program.** Editing, compositing (Fusion), color, audio (Fairlight) and - delivery as "pages" of the same app, on the same project. -3. **Two editing philosophies.** The **Cut** page is fast and lean; the **Edit** page is the full traditional NLE. -4. **Pricing model.** A very capable free version, and Studio as a **one-time purchase** (about US$ 299). -5. **It runs natively on Linux**, which none of the big competitors do. +The [current Blackmagic product page](https://www.blackmagicdesign.com/products/davinciresolve/techspecs) +identifies Resolve 21 and editing, Cut, Color, Fusion and Fairlight contexts. It is product +information, not a benchmark or specification of private engine internals. This audit did +not verify the former 21.1/MCP/API counts or release chronology; those claims are withdrawn. -On the other hand: a steep learning curve, demanding hardware, and **on Linux the free version -does not decode H.264/H.265 or AAC**. That last point is very relevant for OmaMovie. +Codec evidence below comes from the **Resolve 20, July 2025** matrix. Do not silently apply +it to Resolve 21, different editions, containers or future updates. ---- +## 3. Linux codec evidence -## 2. History +[Blackmagic's supported-codec PDF](https://documents.blackmagicdesign.com/SupportNotes/DaVinci_Resolve_20_Supported_Codec_List.pdf), +printed pp. 11 and 14, under Rocky Linux 8.6 (CUDA): -**(general knowledge, unless stated)** - -| Year | Milestone | -|---|---| -| 1980s–2000s | da Vinci Systems builds color correction systems for telecine | -| 2009 | Blackmagic Design buys da Vinci | -| ~2011 | Free (Lite) version released | -| 2014 (v11) | Becomes a full editor (Edit page) | -| 2017 (v14) | Fairlight (audio) integrated | -| 2018 (v15) | Fusion (node compositing) integrated | -| 2019 (v16) | **Cut** page | -| 2020-11 | **Speed Editor** (dedicated editing keyboard) | -| v18 | "Databases" renamed "Project Libraries"; Blackmagic Cloud | -| v18.5 | Native **OpenTimelineIO** support | -| 2025-05 (v20) | 100+ new features, including AI tools | -| 2026-06 (v21) | **Photo** page, more AI tools | -| v21.1 | **MCP** support and 20 new scripting APIs | - ---- - -## 3. Pages: one app, several contexts - -| Page | Function | -|---|---| -| Media | Import and organization | -| **Cut** | Fast assembly | -| **Edit** | Full traditional editing | -| **Fusion** | Node-based compositing and VFX | -| **Color** | Node-based grading | -| **Fairlight** | Audio mixing and mastering | -| Deliver | Export | -| Photo (v21) | Photo editing with Resolve's color tools | - -Every page works on the same timeline and project. There is no export/import between apps, -unlike Premiere + After Effects + Audition, and that is cited as a reason for switching (see -`premiere-pro.md` §10). - -### Implications for OmaMovie - -- Pages are a third way to organize complexity, alongside Premiere's workspaces and iMovie's - contextual inspector. They work for professional workflows in phases (assemble → grade → - mix), but mean context switches and a dense interface on every page. -- For OmaMovie the contextual inspector stays the main model. A future "focus mode" (e.g. - full-screen color) may exist, but as an extension of the inspector, not an app inside the app. -- The strongest architectural lesson: **a single project model for every function**. Color, - audio and compositing read and write the same model. OmaMovie already follows this - (`timeline` → render graph); do not create parallel models per function. - ---- - -## 4. The Cut page: speed for professionals - -Introduced in v16 for fast assemblies on short deadlines: - -- **Source tape**: one button shows every clip in the bin in the viewer as **a single - continuous tape**. You can scrub through all the material and edit without hunting clip by clip. -- **Dual timeline**: the upper timeline shows **the whole program**; the lower one shows **the - working region** zoomed in. Both are editable. Blackmagic justifies it by saying that zooming - and scrolling the timeline is slow. -- A lean interface with fewer visible tools. -- Designed for the **Speed Editor** (§9). - -### Implications for OmaMovie - -- Blackmagic itself, owner of the most "complete" editor, created a simplified page. This - confirms **simple and fast are not opposites**: the Cut page is simple and made for - professionals on a deadline. -- The **dual timeline** is an alternative to Movie Maker's semantic zoom (`movie-maker.md` §4). - Both solve the same problem (overview + detail without constant zooming). Evaluate in the - TimelineView prototype: a compact overview bar (minimap-style) above the timeline may give - the benefit without duplicating the interface. -- **Source tape** is a way to browse the library with the same skimming engine (`imovie.md` §6). - Same technical requirement: fast seek and a thumbnail cache. - ---- - -## 5. Color - -- **Node-based grading**: each node is one correction step; nodes are connected, reordered and - changed without affecting the rest of the grade. Non-destructive by construction. -- Tools: color wheels, curves, **qualifiers** (selection by color/luminance), **power windows** (tracked masks). -- v21: viewing nodes as a **layer list** (layer-list node graph), up to 8-layer stacks per node, - improved ACES workflows, group grading with versions. -- Independent HDR trims for Dolby Vision, HDR10+ and HDR Vivid (21.1). - -### Implications for OmaMovie - -- **Resolve v21 added a layer-list view for the node graph.** Even for colorists, a pure graph - is not always the best interface. For OmaMovie: **render graph as a DAG internally, a list of - layers/effects in the UI**. The graph stays in the core; the UI shows the stack. -- Full node-based grading is an initial non-goal. But the effect interface should allow a color - effect to have several internal stages, so that door stays open. -- ACES and HDR reinforce ADR-0006 (working color space), in line with Premiere 25.2. - ---- - -## 6. Fusion and Fairlight - -- **Fusion**: node-based compositing integrated as a page. v20 brought more advanced multi-layer - compositing; v21/21.1 added the Krokodove toolset (70+ graphics, 3D and procedural tools). -- **Fairlight**: a complete audio workstation inside the editor; v20 brought IntelliCut (removes - silence and splits dialogue between speakers); v21 brought folder tracks. - -### Implications for OmaMovie - -- After Effects/Fusion-style compositing is a **non-goal**. Fusion shows the category's ceiling, - not a requirement. -- Audio: silence removal is a concrete, useful function for spoken content (podcasts, - tutorials). It can be done with audio level analysis, without AI. A low-cost future candidate. - ---- - -## 7. Projects: a database, not a file - -- Projects live in a **Project Library** (formerly "Database"): - - local: a structure managed by Resolve on disk; - - network: **PostgreSQL** (through the "DaVinci Resolve Project Server", a bundled PostgreSQL); - - **Blackmagic Cloud**: hosted libraries for collaboration. -- To get a portable file you must **export a `.drp`**, which contains timelines, edits, grades - and settings, **without media**. At the destination Resolve relinks. -- Forums report crashes and failures when copying libraries to PostgreSQL. - -### Implications for OmaMovie - -- The database model is great for studio collaboration but adds friction for individual users: - the project is not a file you can copy, version or send. -- **It reinforces the `CLAUDE.md` §14/§17 decision**: the project file is the source of truth; - SQLite only for indexes/cache/metadata. An OmaMovie project should be copyable, versionable - in git and attachable to an email. -- `.drp` is a potential import target. Investigate the format before prioritizing it (`Docs/formats/`). - ---- - -## 8. Interoperability - -- **Native OpenTimelineIO since v18.5**: import/export of `.otio` (timeline metadata only) and - `.otioz` (timeline plus media). -- XML, AAF and EDL **(general knowledge)**. - -### Implications for OmaMovie - -- **OTIO is the first interop target.** Resolve (and others) export OTIO natively. That enables - a Resolve → OmaMovie path **without reverse engineering**, validating `Importer → ProjectIR` - with an open, documented format. -- `.otioz` (timeline + media) is a good model for an eventual "export packaged project" in OmaMovie. - ---- - -## 9. Dedicated hardware: Speed Editor - -- An editing keyboard made for the Cut page (released in November 2020). -- A heavy metal **search dial** to move through the timeline quickly; with a trim button held, - the dial becomes a **real-time trim** control. -- Shuttle/jog/scroll buttons change the dial's mode; one key per editing function. -- USB-C or Bluetooth. - -### Implications for OmaMovie - -- Shows the value of mouse-free editing: the same principle as J/K/L in `CLAUDE.md` §11.2. -- The central action system must be **independent of the input device**: keyboard today, - jog/shuttle controllers (USB HID/MIDI) in the future, mapped to the same actions. Do not - implement it now; just do not couple actions to keyboard events. - ---- - -## 10. Scripting, AI and MCP - -- Scripting API (Python/Lua) **(general knowledge)**. 21.1 added 20 APIs (render presets, media - pool, multicam, timeline properties, audio normalization) and **dropped Python 2**. -- **MCP (21.1)**: assistants such as Claude and ChatGPT Codex drive Resolve in natural language - (analyze projects, organize media, change settings, batch render, build highlight reels). -- v20 AI: IntelliScript (timeline from a script), animated captions, Multicam SmartSwitch - (angle switching by who is speaking), Magic Mask, depth map. -- v21 AI: IntelliSearch, CineFocus, facial refinement. - -### Implications for OmaMovie - -- The timeline command system (`CLAUDE.md` §10: every edit is a command) is the natural base for - scripting and automation. A well-defined command serves the UI, the CLI, scripts and an - eventual MCP server. -- **Do not implement scripting now.** Just keep commands serializable and free of UI - dependencies, which undo/redo and tests already require. - ---- - -## 11. Resolve on Linux - -Highly relevant, because it is the professional editor an Omarchy user would use today. - -| Aspect | Situation | +| Entry inspected | Supported conclusion within that table | |---|---| -| Official distro | A **Rocky Linux 8.6**–based environment; Ubuntu/Mint work in practice | -| GPU | **NVIDIA with proprietary drivers** is the only officially supported path (CUDA ≥ 12.8, ≥ 4 GB VRAM). AMD with AMDGPU Pro works, but CUDA-only effects are slow. Intel is not mentioned | -| H.264/H.265 (Free) | **Decode and encode not supported** (licensing) | -| H.264/H.265 (Studio) | Decode supported; **encode only on NVIDIA** | -| AAC | **Not supported on Linux**, not even in Studio | -| Audio | ALSA documented; PipeWire not mentioned | -| Common workaround | Transcode with FFmpeg to DNxHR and remux audio to PCM before importing | - -### Implications for OmaMovie - -**This is the biggest opportunity found in the research.** On Omarchy the typical user has phone -video (H.264/HEVC + AAC), often an AMD or Intel GPU, and PipeWire. The free Resolve cannot open -that material without transcoding, and support outside NVIDIA is partial. - -OmaMovie can take exactly that space: -- H.264/HEVC/AV1 + AAC through FFmpeg and VA-API, **without transcoding**; -- **Intel and AMD as first-class citizens** (VA-API/Vulkan Video), not only NVIDIA; -- native PipeWire; -- packaging for Arch instead of "supported on Rocky Linux". - -This reinforces the priorities in `CLAUDE.md` §7.3. Hardware tests must cover AMD and Intel, not only NVIDIA. - ---- - -## 12. Weaknesses and criticism - -- **A steep learning curve** and a dense interface; professional defaults scare beginners. -- **Demanding hardware**; frequent complaints about performance and crashes on modest machines. -- Free version limits on resolution, effects, codecs and GPU use. -- Linux: codec and GPU limitations (§11). +| H.264/H.265 mov/mkv/mp4 | Decode marked Studio only, acceleration on NVIDIA; encode Studio + NVIDIA | +| AAC / m4a | Decode/encode marked unavailable | +| Linux section title | Documents a CUDA configuration; does not prove all other GPU APIs impossible | -### Implications for OmaMovie +This is documentary analysis, not a local Resolve installation test. The old blanket +"NVIDIA only officially supported" and ALSA/PipeWire exclusion are not inferred from a codec +matrix. Edition-specific media restrictions must be rechecked before public comparisons. -- Resolve shows the capability ceiling but does not serve people who want to edit quickly on a - laptop. OmaMovie must **work well on modest hardware** (Intel/AMD integrated GPUs), with - proxies and explicit degradation instead of freezing. -- Benchmarks (`CLAUDE.md` §23) must include at least one machine with an integrated GPU. +## 4. UX and project inference ---- +Cut and Edit suggest that the same product can present editing in different contexts. +For OmaMovie a minimap versus single zoomed timeline is still a testable preference; pages +or a node/layer view do not validate our exact layout or render-graph representation. -## 13. Prioritization table for OmaMovie +Choose file-based project persistence for portability/recovery requirements. Database +storage can also provide transactions and portability via export; no storage form is +inherently unreliable. A database/workspace workflow does not prove that a file design is +universally better. `.drp` import remains uninvestigated, not an accessible-format promise. -| Resolve capability | Adopt? | When | Module | -|---|---|---|---| -| A single project model for every function | Yes (already planned) | From the start | `timeline`, `project` | -| Overview + detail timeline (dual timeline / minimap) | Evaluate in the prototype | Phase 10 | UI | -| Source tape (library as a continuous tape) | Evaluate | After the library | UI, `media` | -| Internal DAG render graph, layer-stack UI | Yes | Phase 5 | `compositor` | -| OTIO import | Yes, first interop target | Phase 12 | `importers` | -| Projects in a database | **No**; the file is the source of truth | — | `project` | -| Silence removal | Future, without AI | After audio | `audio` | -| Actions independent of the input device | Yes (design) | Phase 10 | UI | -| Serializable commands (base for scripting/MCP) | Yes (design) | Phase 9 | `timeline` | -| Scripting/MCP | Not now | Future | — | -| Full Fusion/Fairlight | No | Non-goal | — | -| Consumer codecs + AMD/Intel + PipeWire on Linux | **Yes, a differentiator** | Phases 2–8 | `media`, `audio` | -| `.drp` import | Investigate | Future | `importers` | +## 5. OTIO and fidelity ---- +[OTIO's file specification](https://opentimelineio.readthedocs.io/en/latest/tutorials/otio-file-format-specification.html) +defines a versioned timeline schema. That makes a bounded first importer reasonable. This +review did not read a versioned Blackmagic OTIO export contract or run an export. The prior +"native since 18.5 including otioz" statement is not a validated end-to-end conversion path. +Obtain real, versioned exports and compare cuts, media references, time and unsupported +effects. Public schema accessibility is distinct from lossless application interoperability. -## Sources +## 6. Proposed audience check -- [Blackmagic Design: DaVinci Resolve 20 announcement](https://www.blackmagicdesign.com/media/partial/release/20250404-02) -- [Post Magazine: Blackmagic Design unveils DaVinci Resolve 20](https://www.postmagazine.com/Press-Center/Daily-News/2025/Blackmagic-Design-unveils-DaVinci-Resolve-20-wit.aspx) -- [Post Magazine: Blackmagic Design releases v20 of DaVinci Resolve & Fusion](https://www.postmagazine.com/Press-Center/Daily-News/2025/Blackmagic-Design-releases-V-20-of-DaVinci-Resol.aspx) -- [Newsshooter: DaVinci Resolve 20 announced](https://www.newsshooter.com/2025/04/04/blackmagic-design-davinci-resolve-20-announced-with-100-new-features-including-ai-enhancements/) -- [Broadcast Now: NAB 2025 — AI rough cut and audio generation in Resolve](https://www.broadcastnow.co.uk/production-and-post/nab-2025-blackmagic-adds-ai-powered-rough-cut-and-audio-generation-to-resolve/5203915.article) -- [PetaPixel: DaVinci Resolve 21 officially released](https://petapixel.com/2026/06/03/davinci-resolve-21-officially-released-with-new-photo-editing-ai-tools/) -- [CineD: DaVinci Resolve 21 announced](https://www.cined.com/davinci-resolve-21-announced-new-photo-page-eight-new-ai-tools-tethered-camera-controls-and-more/) -- [Motion Media: DaVinci Resolve 21.1 update](https://www.motionmedia.com/mm-blog/davinci-resolve-21-1-update-new-features) -- [RedShark News: DaVinci Resolve 21.1 new features](https://www.redsharknews.com/davinci-resolve-21.1-new-features-release) -- [Storyblocks: Color grading in DaVinci Resolve](https://www.storyblocks.com/resources/tutorials/davinci-resolve-color-grading) -- [TourBox: A complete guide to nodes in DaVinci Resolve](https://www.tourboxtech.com/en/news/davinci-resolve-nodes.html) -- [Pro Video Coalition: DaVinci Resolve 16](https://www.provideocoalition.com/davinci-resolve-16-adds-lufs-audio-loudness-standards-linear-features/) -- [Wipster: Inside DaVinci Resolve's new Cut page](https://wipster.io/blog/inside-davinci-resolves-new-cut-page) -- [PremiumBeat: DaVinci Resolve 16 Cut page review](https://www.premiumbeat.com/blog/davinci-resolve-16-cut-page-review/) -- [Uncle: DaVinci Resolve Project Library vs database](https://tryuncle.com/learn/davinci-resolve/davinci-resolve-project-library-vs-database) -- [Blackmagic Forum: Copy DB from local to Postgres fails or crashes](https://forum.blackmagicdesign.com/viewtopic.php?p=746517) -- [Prism Pipeline: OpenTimelineIO integration](https://prism-pipeline.com/docs/latest/general/miscellaneous/opentimelineio/) -- [Digital Production: DaVinci Resolve Speed Editor release](https://digitalproduction.com/2020/11/11/davinci-resolve-speed-editor-release/) -- [tal.org: Blackmagic DaVinci Resolve on Linux](https://www.tal.org/tutorials/blackmagic-davinci-resolve-linux) -- [NixOS Wiki: DaVinci Resolve](https://wiki.nixos.org/wiki/DaVinci%20Resolve) -- [Blackmagic Forum: Can't play clips, use CUDA, .mp4 or h.264 in Linux](https://forum.blackmagicdesign.com/viewtopic.php?p=742208) -- [Movavi: How to use DaVinci Resolve (beginner's guide)](https://www.movavi.com/support/how-to/how-to-use-davinci-resolve/) -- [rfp.wiki: DaVinci Resolve vs Final Cut Pro](https://www.rfp.wiki/vendors/davinci-resolve/final-cut-pro) +Baseline Kdenlive/Shotcut and the exact Resolve edition with the same phone/recording files. +Observe import friction, simple edit completion, preview deadlines and output correctness +on integrated graphics. The hypothesis is that local consumer-media editing with low setup +cost is useful on Omarchy. Evidence that existing tools already meet the tasks with similar +friction would weaken the proposed differentiation. No user or competitor benchmark ran. diff --git a/Docs/Research/evidence/2026-10-03-calculations.txt b/Docs/Research/evidence/2026-10-03-calculations.txt new file mode 100644 index 0000000..bfa2c78 --- /dev/null +++ b/Docs/Research/evidence/2026-10-03-calculations.txt @@ -0,0 +1,11 @@ +1920x1080 NV12: 3110400 bytes; 2.966309 MiB; one full-frame transfer at 60 fps: 0.186624 GB/s +1920x1080 P010: 6220800 bytes; 5.932617 MiB; one full-frame transfer at 60 fps: 0.373248 GB/s +1920x1080 RGBA16F: 16588800 bytes; 15.820312 MiB; one full-frame transfer at 60 fps: 0.995328 GB/s +3840x2160 NV12: 12441600 bytes; 11.865234 MiB; one full-frame transfer at 60 fps: 0.746496 GB/s +3840x2160 P010: 24883200 bytes; 23.730469 MiB; one full-frame transfer at 60 fps: 1.492992 GB/s +3840x2160 RGBA16F: 66355200 bytes; 63.281250 MiB; one full-frame transfer at 60 fps: 3.981312 GB/s +60 fps deadline: 16.666666666666668 ms +4K RGBA16F, 3 streams x 8 frames: 1518.75 MiB +10 s constant 2x: 20 s source +10 s linear 1x to 2x: 15.0 s source +10 s freeze: 0 s source diff --git a/Docs/Research/evidence/2026-10-03-local.txt b/Docs/Research/evidence/2026-10-03-local.txt new file mode 100644 index 0000000..58afc5f --- /dev/null +++ b/Docs/Research/evidence/2026-10-03-local.txt @@ -0,0 +1,48 @@ +Audit date: 2026-10-03 +Repository baseline: 2f6abf32e95a10f29ebeb39ac2b67ec36b297397 +Read-only environment observations; no GPU benchmark rerun. + +$ pacman -Q omarchy ffmpeg mesa qt6-base vulkan-headers +omarchy 4.0.4-1 +ffmpeg 2:9.0.1-4 +mesa 1:26.2.2-1 +qt6-base 6.11.2-3 +vulkan-headers 1:1.4.357.0-1 + +exit=0 + +$ pkg-config --modversion libavutil Qt6Quick +61.1.101 +6.11.2 + +exit=0 + +$ ffmpeg -version +ffmpeg version n9.0.1 Copyright (c) 2000-2026 the FFmpeg developers +built with gcc 16 (GCC) +configuration: --prefix=/usr --disable-debug --disable-static --disable-stripping --enable-amf --enable-avisynth --enable-cuda-llvm --enable-lto --enable-fontconfig --enable-frei0r --enable-gmp --enable-gnutls --enable-gpl --enable-ladspa --enable-lcms2 --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libdav1d --enable-libdrm --enable-libdvdnav --enable-libdvdread --enable-libfreetype --enable-libfribidi --enable-libgsm --enable-libharfbuzz --enable-libiec61883 --enable-libjack --enable-libjxl --enable-libmodplug --enable-libmp3lame --enable-libopencore_amrnb --enable-libopencore_amrwb --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libplacebo --enable-libpulse --enable-librav1e --enable-librsvg --enable-librubberband --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libsvtav1 --enable-libtheora --enable-libv4l2 --enable-libvidstab --enable-libvmaf --enable-libvorbis --enable-libvpl --enable-libvpx --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxcb --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-nvdec --enable-nvenc --enable-opencl --enable-opengl --enable-shared --enable-vapoursynth --enable-version3 --enable-vulkan --disable-decoder=magicyuv +libavutil 61. 1.101 / 61. 1.101 +libavcodec 63. 1.101 / 63. 1.101 +libavformat 63. 1.101 / 63. 1.101 +libavdevice 63. 1.101 / 63. 1.101 +libavfilter 12. 1.101 / 12. 1.101 +libswscale 10. 1.101 / 10. 1.101 +libswresample 7. 1.101 / 7. 1.101 + +Exiting with exit code 0 + +exit=0 + +SHA-256 of inspected local sources (installed files, not upstream release provenance): +9d1aaeb0c16104d678742c04d6a8700fed365e7eb65a8eaef5fb3845ddf634e3 /usr/share/omarchy/version +2356e6e19656fcc67aa345fc46352a310945e54146706c039bf92a7bd3b3fb5a /usr/share/omarchy/bin/omarchy-theme-set +a429ed1b18114ff3e784fc4479c6b829e9effb64c0f9d22d3b15eda4c62d86b5 /usr/share/omarchy/bin/omarchy-theme-color +6d70541c44a82fde5a44d712f348df35f8372c080436433916694738627c4c66 /usr/share/omarchy/bin/omarchy-capture-screenrecording +fa35b43154b49a286c2717490a07b542b5868b3e5101e86e3df628b22bc55962 /usr/share/omarchy/default/hypr/windows.lua +e2409d80ef7e88455326e50f853d60bb18b2cf909b5820066bf46d538aaeedd7 /usr/share/omarchy/default/hypr/apps/davinci-resolve.lua +99434167d84b242050bbb32485679e92c29934fd5102df01265f7673612167fc /usr/share/omarchy/install/omarchy-base.packages +036d13f9b4e74653fbf18f30b867b5ec3397f39d87b97d0400aac3378d6a6603 /usr/share/omarchy/install/hardware/vulkan.sh +1c796829df784813ad53df48c967eb17281054a9b0ff544949be2ad3a1bcb946 /usr/share/omarchy/install/hardware/intel/video-acceleration.sh +aeb211e2a4c97eac7fd2ff8908e57b075dc2a125f66a2fe47e770c8cf7cc2634 /usr/include/libavutil/hwcontext_vulkan.h + +/usr/share/omarchy/version: 4.0.0.alpha diff --git a/Docs/Research/evidence/2026-10-03-make-test.txt b/Docs/Research/evidence/2026-10-03-make-test.txt new file mode 100644 index 0000000..4459c2e --- /dev/null +++ b/Docs/Research/evidence/2026-10-03-make-test.txt @@ -0,0 +1,960 @@ +Command: make test +Mode: debug, executed outside sandbox after sandbox GPU initialization failure +Exit status: 0 +Environment: 2026-10-03-local.txt +These are existing tests, not the proposed validation pilots. GPU-dependent branches may be conditional. + + +== base_tests (debug) + +● Rational + normalizes sign and common factors + ✓ r.has_value() to be truthy + ✓ r->num() to equal -3 + ✓ r->den() to equal 4 + rejects a zero denominator + ✓ r.has_value() to be falsy + ✓ static_cast(r.error().code()) to equal static_cast(ErrorCode::InvalidArgument) + rejects INT64_MIN because it cannot be negated + ✓ Rational::make(kMin, 1).has_value() to be falsy + ✓ Rational::make(1, kMin).has_value() to be falsy + orders fractions exactly + ✓ Rational::literal(1, 3) < Rational::literal(1, 2) to be truthy + ✓ Rational::literal(-1, 2) < Rational::literal(1, 3) to be truthy + ✓ Rational::literal(30000, 1001) > Rational::literal(29, 1) to be truthy + ✓ Rational::literal(kMax, 1) > Rational::literal(kMax - 1, 1) to be truthy + ✓ Rational::literal(kMax, kMax - 1) > Rational::literal(kMax - 1, kMax - 2) to be falsy + inverts and refuses to invert zero + ✓ Rational::literal(1001, 30000).inverse()->num() to equal 30000 + ✓ Rational::literal(0, 1).inverse().has_value() to be falsy + +● rescale + converts seconds to a 90 kHz timebase exactly + ✓ rescale_ok(1, kSecond, kTb90k, Rounding::Floor) to equal 90000 + converts one 29.97 fps frame to 90 kHz exactly (3003 ticks) + ✓ rescale_ok(1, kFrame2997, kTb90k, Rounding::Floor) to equal 3003 + rounds one 23.976 fps frame (3753.75 ticks) as requested + ✓ rescale_ok(1, kFrame23976, kTb90k, Rounding::Floor) to equal 3753 + ✓ rescale_ok(1, kFrame23976, kTb90k, Rounding::Ceil) to equal 3754 + ✓ rescale_ok(1, kFrame23976, kTb90k, Rounding::Nearest) to equal 3754 + rounds negative values toward the right direction + ✓ rescale_ok(-1, kFrame23976, kTb90k, Rounding::Floor) to equal -3754 + ✓ rescale_ok(-1, kFrame23976, kTb90k, Rounding::Ceil) to equal -3753 + ✓ rescale_ok(-1, kFrame23976, kTb90k, Rounding::Nearest) to equal -3754 + breaks ties away from zero with Nearest + ✓ rescale_ok(1, Rational::literal(1, 2), kSecond, Rounding::Nearest) to equal 1 + ✓ rescale_ok(-1, Rational::literal(1, 2), kSecond, Rounding::Nearest) to equal -1 + ✓ rescale_ok(3, Rational::literal(1, 2), kSecond, Rounding::Nearest) to equal 2 + maps video frames to audio samples (29.97 fps at 48 kHz = 1601.6 samples) + ✓ rescale_ok(1, kFrame2997, kTb48k, Rounding::Nearest) to equal 1602 + ✓ rescale_ok(5, kFrame2997, kTb48k, Rounding::Nearest) to equal 8008 + ✓ rescale_ok(1, kFrame2997, Rational::literal(1, 44100), Rounding::Floor) to equal 1471 + handles ten hours of media in a 90 kHz timebase + ✓ rescale_ok(ten_hours, kTb90k, kTb48k, Rounding::Floor) to equal 10LL * 3600 * 48000 + ✓ rescale_ok(ten_hours, kTb90k, kFrame2997, Rounding::Floor) to equal 1078921 + reports overflow instead of truncating + ✓ static_cast(rescale_error(kMax, kSecond, kTb90k)) to equal static_cast(ErrorCode::Overflow) + ✓ rescale_ok(kMax, kTb90k, kSecond, Rounding::Floor) to equal kMax / 90000 + rejects non-positive units + ✓ static_cast(rescale_error(1, Rational::literal(0, 1), kSecond)) to equal static_cast(ErrorCode::InvalidArgument) + ✓ static_cast(rescale_error(1, kSecond, Rational::literal(-1, 25))) to equal static_cast(ErrorCode::InvalidArgument) + +● RationalTime + compares instants exactly across timebases + ✓ t(3003, kTb90k) == t(1001, kTb30k) to be truthy + ✓ t(48000, kTb48k) == t(90000, kTb90k) to be truthy + ✓ t(1, kTb48k) < t(2, kTb90k) to be truthy + ✓ t(-1, kTb48k) < t(0, kTb90k) to be truthy + compares extreme values without overflowing + ✓ t(max, kSecond) > t(max, Rational::literal(1, 2)) to be truthy + ✓ t(max, kTb90k) < t(max, kTb48k) to be truthy + ✓ t(max - 1, Rational::literal(1, max)) < t(1, kSecond) to be truthy + rejects a non-positive timebase + ✓ RationalTime::make(1, Rational::literal(0, 1)).has_value() to be falsy + ✓ RationalTime::make(1, Rational::literal(-1, 25)).has_value() to be falsy + rescales with explicit rounding + ✓ r->value() to equal 3754 + ✓ r->timebase() == kTb90k to be truthy + adds only within one timebase + ✓ t(10, kTb90k).plus(t(5, kTb90k))->value() to equal 15 + ✓ code_of(bad.error()) to equal static_cast(ErrorCode::InvalidArgument) + detects overflow in addition + ✓ code_of(r.error()) to equal static_cast(ErrorCode::Overflow) + +● TimeRange + is half-open + ✓ range->contains(t(100, kTb90k)) to be truthy + ✓ range->contains(t(149, kTb90k)) to be truthy + ✓ range->contains(t(150, kTb90k)) to be falsy + ✓ range->contains(t(99, kTb90k)) to be falsy + ✓ range->end().value() to equal 150 + checks membership across timebases + ✓ range->contains(t(89999, kTb90k)) to be truthy + ✓ range->contains(t(90000, kTb90k)) to be falsy + detects overlap + ✓ a->overlaps(*b) to be falsy + ✓ a->overlaps(*c) to be truthy + rejects negative durations and mixed timebases + ✓ TimeRange::make(t(0, kTb90k), t(-1, kTb90k)).has_value() to be falsy + ✓ TimeRange::make(t(0, kTb90k), t(1, kTb48k)).has_value() to be falsy + +● FrameRate + represents NTSC rates as exact fractions + ✓ fr::k29_97.fps() == Rational::literal(30000, 1001) to be truthy + ✓ fr::k59_94.frame_duration() == Rational::literal(1001, 60000) to be truthy + maps frame 300 at 29.97 fps to 10.01 s + ✓ at == t(1001, Rational::literal(1, 100)) to be truthy + maps a time back to a frame with explicit rounding + ✓ *fr::k29_97.time_to_frame(t(10, kSecond), Rounding::Floor) to equal 299 + ✓ *fr::k29_97.time_to_frame(t(10, kSecond), Rounding::Nearest) to equal 300 + ✓ *fr::k25.time_to_frame(t(10, kSecond), Rounding::Floor) to equal 250 + rejects non-positive rates + ✓ FrameRate::make(Rational::literal(0, 1)).has_value() to be falsy + +● SampleRate + addresses audio in samples + ✓ *sr->time_to_sample(t(1, kSecond), Rounding::Floor) to equal 48000 + ✓ sr->sample_to_time(24000) == t(1, Rational::literal(1, 2)) to be truthy + converts 44.1 kHz positions to 96 kHz + ✓ *sr96->time_to_sample(pos, Rounding::Floor) to equal 96000 + rejects non-positive rates + ✓ SampleRate::make(0).has_value() to be falsy + ✓ SampleRate::make(-48000).has_value() to be falsy + +● Error and Result + carries code, category, message and context + ✓ static_cast(e.code()) to equal static_cast(ErrorCode::InvalidData) + ✓ static_cast(e.category()) to equal static_cast(Category::Importer) + ✓ e.message() to equal "truncated header" + ✓ e.context() to equal "clip.prproj" + describes itself for logs and UI warnings + ✓ e.summary() to equal "[timeline] overflow: trim too long (clip 4)" + ✓ bare.summary() to equal "[base] cancelled: stopped" + propagates through Result + ✓ *parse_positive(3) to equal 3 + ✓ r.has_value() to be falsy + ✓ r.error().context() to equal "field 'duration'" + +● Logging + formats messages and records category and call site + ✓ record_count() to equal 1 + ✓ c.message to equal "opened clip.mp4 (2 streams)" + ✓ static_cast(c.category) to equal static_cast(Category::Decode) + ✓ static_cast(c.level) to equal static_cast(LogLevel::Info) + ✓ c.line to be greater than 0 + drops records below the category level + ✓ record_count() to equal 0 + ✓ record_count() to equal 1 + reports enabled levels without formatting + ✓ oma::log_enabled(Category::Cache, LogLevel::Info) to be falsy + ✓ oma::log_enabled(Category::Cache, LogLevel::Error) to be truthy + ✓ oma::log_enabled(Category::Cache, LogLevel::Off) to be falsy + +● JobPool + runs a job and reports success with full progress + ✓ h.wait().has_value() to be truthy + ✓ value.load() to equal 42 + ✓ state_of(h) to equal st(JobState::Succeeded) + ✓ h.progress() to be close to 1 (precision 0.0001) + keeps the error of a failed job + ✓ wait_code(h) to equal static_cast(ErrorCode::InvalidData) + ✓ state_of(h) to equal st(JobState::Failed) + never starts a pending job that was cancelled + ✓ first.wait().has_value() to be truthy + ✓ wait_code(second) to equal static_cast(ErrorCode::Cancelled) + ✓ state_of(second) to equal st(JobState::Cancelled) + ✓ second_ran.load() to be falsy + cancels a running job cooperatively + ✓ wait_code(h) to equal static_cast(ErrorCode::Cancelled) + ✓ state_of(h) to equal st(JobState::Cancelled) + shuts down with running and pending jobs + ✓ state_of(running) to equal st(JobState::Cancelled) + ✓ state_of(pending) to equal st(JobState::Cancelled) + ✓ pending_ran.load() to be falsy + refuses work after shutdown + ✓ state_of(h) to equal st(JobState::Cancelled) + turns an escaping exception into an internal error + ✓ wait_code(h) to equal static_cast(ErrorCode::Internal) + ✓ state_of(h) to equal st(JobState::Failed) + runs many jobs across workers + ✓ counter.load() to equal 200 + +Test Suites Summary: + All 109 tests passed! + + +== gpu_tests (debug) + +● gpu::Device +15:19:52.251 INFO gpu device: Intel(R) Iris(R) Xe Graphics (TGL GT2) (Intel open-source Mesa driver, Mesa 26.2.2-arch1.1), Vulkan 1.4, 8 extensions, internally synchronized queues: yes (device.cpp:280) + reports the selected device + ✓ d->info().name.empty() to be falsy + ✓ d->info().api_version >= VK_API_VERSION_1_3 to be truthy + device: Intel(R) Iris(R) Xe Graphics (TGL GT2) (Mesa 26.2.2-arch1.1) + exposes valid raw handles for interop + ✓ d->instance() != VK_NULL_HANDLE to be truthy + ✓ d->physical_device() != VK_NULL_HANDLE to be truthy + ✓ d->device() != VK_NULL_HANDLE to be truthy + ✓ d->instance_proc_addr() != nullptr to be truthy + has a graphics and compute queue family + ✓ family_has(*d, g, VK_QUEUE_GRAPHICS_BIT) to be truthy + ✓ family_has(*d, g, VK_QUEUE_COMPUTE_BIT) to be truthy + ✓ d->queue(g, 0) != VK_NULL_HANDLE to be truthy + enables timeline semaphores and synchronization2 + ✓ timeline to be truthy + ✓ sync2 to be truthy + only reports extensions it enabled + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension(e) to be truthy + ✓ d->has_extension("VK_NOT_A_REAL_extension") to be falsy + matches the queue creation flags to internal synchronization + ✓ flagged == d->internally_synchronized_queues() to be truthy + locks and unlocks every queue it created + ✓ true to be truthy + +● gpu::enumerate_devices + lists the device that was created + ✓ all.has_value() to be truthy + ✓ found to be truthy + rejects a device name that does not exist + ✓ r.has_value() to be falsy + ✓ static_cast(r.error().code()) to equal static_cast(ErrorCode::Unsupported) + +● gpu::Buffer and gpu::CommandRunner + round-trips bytes through device-local memory + ✓ upload && local && readback && runner to be truthy + ✓ upload->mapped() != nullptr to be truthy + ✓ local->mapped() == nullptr to be truthy + ✓ ran.has_value() to be truthy + ✓ std::memcmp(readback->mapped(), pattern.data(), pattern.size()) == 0 to be truthy + creates images with views and moves them + ✓ img.has_value() to be truthy + ✓ img->handle() != VK_NULL_HANDLE to be truthy + ✓ img->view() != VK_NULL_HANDLE to be truthy + ✓ img->desc().width to equal 64 + ✓ moved.handle() != VK_NULL_HANDLE to be truthy + ✓ img->handle() == VK_NULL_HANDLE to be truthy + +Test Suites Summary: + All 37 tests passed! + + +== media_tests (debug) + +● media::probe + describes an H.264 + AAC file + ✓ info.has_value() to be truthy + ✓ info->container to contain "mp4" + ✓ info->streams.size() to equal 2 + ✓ info->duration.has_value() to be truthy + ✓ info->duration->seconds_approx() to be close to 1 (precision 0.1) + ✓ v != nullptr to be truthy + ✓ v->kind == StreamKind::Video to be truthy + ✓ v->codec to equal "h264" + ✓ v->video->width to equal 320 + ✓ v->video->height to equal 180 + ✓ v->video->pixel_format to equal "yuv420p" + ✓ v->video->bit_depth to equal 8 + ✓ v->video->rotation to equal 0 + ✓ v->video->frame_rate.has_value() to be truthy + ✓ v->video->frame_rate->fps() == Rational::literal(30, 1) to be truthy + ✓ v->video->variable_frame_rate to be falsy + ✓ v->video->still_image to be falsy + ✓ v->frame_count.value_or(0) to equal 30 + ✓ a != nullptr to be truthy + ✓ a->codec to equal "aac" + ✓ a->audio->sample_rate.has_value() to be truthy + ✓ a->audio->sample_rate->hz() to equal 48000 + ✓ a->audio->channels to equal 1 + ✓ a->audio->channel_layout to equal "mono" + keeps NTSC frame rates exact + ✓ info.has_value() to be truthy + ✓ rate.has_value() to be truthy + ✓ rate && rate->fps() == Rational::literal(30000, 1001) to be truthy + detects variable frame rate from timestamps + ✓ info.has_value() to be truthy + ✓ best_video(*info)->video->variable_frame_rate to be truthy + reads the display rotation + ✓ info.has_value() to be truthy + ✓ best_video(*info)->video->rotation to equal 90 + reports 10-bit HEVC + ✓ info.has_value() to be truthy + ✓ v.codec to equal "hevc" + ✓ v.profile to equal "Main 10" + ✓ v.video->bit_depth to equal 10 + ✓ v.video->color.range == ColorRange::Full to be falsy + describes audio-only files + ✓ info.has_value() to be truthy + ✓ info->best_video.has_value() to be falsy + ✓ a != nullptr to be truthy + ✓ a->audio->sample_rate->hz() to equal 44100 + ✓ a->duration.has_value() to be truthy + ✓ a->duration->seconds_approx() to be close to 2 (precision 0.05) + marks still images + ✓ info.has_value() to be truthy + ✓ best_video(*info)->video->still_image to be truthy + ✓ best_video(*info)->video->width to equal 640 + rejects a truncated file as invalid data + ✓ info.has_value() to be falsy + ✓ code_of(info.error()) to equal static_cast(ErrorCode::InvalidData) + reports a missing file as an I/O error + ✓ info.has_value() to be falsy + ✓ code_of(info.error()) to equal static_cast(ErrorCode::IoError) + ✓ info.error().context() to contain "does-not-exist.mp4" + +● media::VideoDecoder (software) + decodes every frame in presentation order + ✓ d.has_value() to be truthy + ✓ d->path() == DecodePath::Software to be truthy + ✓ d->stream().codec to equal "h264" + ✓ f.has_value() to be truthy + ✓ frame.on_gpu() to be falsy + ✓ frame.width() to equal 320 + ✓ frame.height() to equal 180 + ✓ std::string(frame.pixel_format()) to equal "yuv420p" + ✓ frame.plane_count() to equal 3 + ✓ frame.layout().chroma_shift_x to equal 1 + ✓ frame.layout().chroma_shift_y to equal 1 + ✓ frame.layout().interleaved_chroma to be falsy + ✓ frame.layout().container_bits to equal 8 + ✓ frame.stride(0) >= 320 to be truthy + ✓ frame.plane(0).size() >= static_cast(frame.stride(0)) * 180 to be truthy + ✓ frame.pts().has_value() to be truthy + ✓ frame.pts()->value() to equal 0 + ✓ frame.acquire_gpu().has_value() to be falsy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ count to equal 30 + ✓ increasing to be truthy + ✓ after_end.has_value() && !after_end->has_value() to be truthy + keeps 10-bit samples + ✓ d.has_value() to be truthy + ✓ f.has_value() && f->has_value() to be truthy + ✓ std::string((*f)->pixel_format()) to equal "yuv420p10le" + ✓ (*f)->bit_depth() to equal 10 + ✓ (*f)->layout().container_bits to equal 16 + ✓ (*f)->layout().lsb_shift to equal 0 + seeks to the frame on screen on VFR media + ✓ d.has_value() to be truthy + ✓ pts.size() > 10 to be truthy + ✓ d->seek(ticks(*d, target)).has_value() to be truthy + ✓ f.has_value() && f->has_value() to be truthy + ✓ (*f)->pts()->value() to equal pts[i] + ✓ following.has_value() && following->has_value() to be truthy + ✓ (*following)->pts()->value() to equal pts[i + 1] + ✓ d->seek(ticks(*d, target)).has_value() to be truthy + ✓ f.has_value() && f->has_value() to be truthy + ✓ (*f)->pts()->value() to equal pts[i] + ✓ following.has_value() && following->has_value() to be truthy + ✓ (*following)->pts()->value() to equal pts[i + 1] + ✓ d->seek(ticks(*d, target)).has_value() to be truthy + ✓ f.has_value() && f->has_value() to be truthy + ✓ (*f)->pts()->value() to equal pts[i] + ✓ following.has_value() && following->has_value() to be truthy + ✓ (*following)->pts()->value() to equal pts[i + 1] + seeks to exact frames, before the start and past the end + ✓ d.has_value() to be truthy + ✓ d->seek(ticks(*d, pts[5])).has_value() to be truthy + ✓ on && *on && (*on)->pts()->value() == pts[5] to be truthy + ✓ d->seek(ticks(*d, -1000)).has_value() to be truthy + ✓ first && *first && (*first)->pts()->value() == pts.front() to be truthy + ✓ d->seek(ticks(*d, pts.back() + 10000)).has_value() to be truthy + ✓ last && *last && (*last)->pts()->value() == pts.back() to be truthy + ✓ end.has_value() && !end->has_value() to be truthy + seeks with a time in another timebase + ✓ d.has_value() to be truthy + ✓ d->seek(t).has_value() to be truthy + ✓ f && *f && (*f)->pts().has_value() to be truthy + ✓ *(*f)->pts() == t to be truthy + fails without a usable path + ✓ d.has_value() to be falsy + ✓ code_of(d.error()) to equal static_cast(ErrorCode::Unsupported) + rejects damaged files and non-video streams + ✓ truncated.has_value() to be falsy + ✓ wrong.has_value() to be falsy + ✓ code_of(wrong.error()) to equal static_cast(ErrorCode::InvalidArgument) + +● media::VideoDecoder (GPU paths) + decodes H.264 onto OmaMovie's device + ✓ d.has_value() to be truthy + ✓ consumer.ok() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ frame.layout().interleaved_chroma to be truthy + ✓ frame.layout().lsb_shift to equal bit_depth > 8 ? 16 - bit_depth : 0 + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + h264_30fps_aac.mp4: VA-API to Vulkan, 30 frames, 30 on the GPU + ✓ frames to equal 30 + ✓ on_gpu to equal frames + ✓ consumer.consumed() to equal frames + ✓ same_luma(gpu_luma, software_luma(name), bit_depth) to be truthy + decodes 10-bit HEVC onto OmaMovie's device + ✓ d.has_value() to be truthy + ✓ consumer.ok() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ frame.layout().interleaved_chroma to be truthy + ✓ frame.layout().lsb_shift to equal bit_depth > 8 ? 16 - bit_depth : 0 + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + hevc_10bit.mp4: VA-API to Vulkan, 30 frames, 30 on the GPU + ✓ frames to equal 30 + ✓ on_gpu to equal frames + ✓ consumer.consumed() to equal frames + ✓ same_luma(gpu_luma, software_luma(name), bit_depth) to be truthy + decodes AV1 onto OmaMovie's device + ✓ d.has_value() to be truthy + ✓ consumer.ok() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ frame.layout().interleaved_chroma to be truthy + ✓ frame.layout().lsb_shift to equal bit_depth > 8 ? 16 - bit_depth : 0 + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + ✓ frame.path() == (*d)->path() to be truthy + ✓ f.has_value() to be truthy + av1_opus.mkv: VA-API to Vulkan, 30 frames, 30 on the GPU + ✓ frames to equal 30 + ✓ on_gpu to equal frames + ✓ consumer.consumed() to equal frames + ✓ same_luma(gpu_luma, software_luma(name), bit_depth) to be truthy + falls back to software when the driver refuses a stream + ✓ d.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + ✓ f.has_value() to be truthy + h264_444.mp4: software + ✓ frames to equal 30 + ✓ first_pts to equal 0 + reports a refusal when software is not allowed + ✓ (f && *f && (*f)->on_gpu()) || (!f && f.error().code() == oma::ErrorCode::Unsupported) to be truthy + +● media::AudioDecoder + decodes PCM to planar float with contiguous timestamps + ✓ d.has_value() to be truthy + ✓ (*d)->sample_rate().hz() to equal 44100 + ✓ (*d)->channels() to equal 1 + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ t.frames to equal 88200 + ✓ t.first_pts to equal 0 + ✓ t.contiguous to be truthy + ✓ static_cast(t.peak) to be close to 0.125 (precision 0.01) + resamples and changes the channel count + ✓ d.has_value() to be truthy + ✓ (*d)->sample_rate().hz() to equal 48000 + ✓ (*d)->channels() to equal 2 + ✓ first && *first && (*first)->channel(1).size() == (*first)->channel(0).size() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ total to equal 96000 + decodes AAC and Opus + ✓ aac.has_value() to be truthy + ✓ (*aac)->stream().codec to equal "aac" + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ std::llabs(t.frames - 48000) <= 2048 to be truthy + ✓ t.contiguous to be truthy + ✓ opus.has_value() to be truthy + ✓ (*opus)->stream().codec to equal "opus" + ✓ (*opus)->sample_rate().hz() to equal 48000 + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ std::llabs(t.frames - 48000) <= 2048 to be truthy + seeks to an exact sample + ✓ (*d)->seek(t).has_value() to be truthy + ✓ b && *b to be truthy + ✓ (*b)->pts.value() to equal 30000 + ✓ static_cast((*b)->channel(0)[0]) to be close to 0.113194 (precision 0.001) + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ b.has_value() to be truthy + ✓ first_frames + rest.frames to equal 88200 - 30000 + fails on files without audio + ✓ d.has_value() to be falsy + ✓ code_of(d.error()) to equal static_cast(ErrorCode::InvalidArgument) + +Test Suites Summary: + All 571 tests passed! + + +== compositor_tests (debug) + +● compositor geometry + composes and inverts affine transforms + ✓ maps(m, 1, 0, 10, 22) to be truthy + ✓ inv.has_value() to be truthy + ✓ maps(*inv, 10, 22, 1, 0) to be truthy + ✓ maps(m.then(*inv), 5, -7, 5, -7, 1e-12) to be truthy + ✓ Affine::scale(0, 1).inverse().has_value() to be falsy + fits, fills and places sources natively + ✓ maps(m, 0, 0, 0, 0) && maps(m, 320, 180, 1920, 1080) to be truthy + ✓ maps(m, 0, 0, 240, 0) && maps(m, 640, 480, 1680, 1080) to be truthy + ✓ maps(m, 0, 0, 0, -180) && maps(m, 640, 480, 1920, 1260) to be truthy + ✓ maps(m, 0, 0, 800, 450) to be truthy + applies display rotation and pixel aspect + ✓ box.has_value() to be truthy + ✓ (*box)[0] to equal 656 + ✓ (*box)[2] to equal 1264 + ✓ (*box)[1] to equal 0 + ✓ (*box)[3] to equal 1080 + ✓ maps(m, 0, 0, 656.25, 1080) to be truthy + ✓ maps(w, 160, 180, 1920, 1080) to be truthy + crops, scales and offsets layers + ✓ maps(m, 200, 100, 600, 450) to be truthy + ✓ maps(m, 100, 100, 400, 450) to be truthy + ✓ box && (*box)[0] == 400 && (*box)[2] == 800 && (*box)[1] == 250 && (*box)[3] == 650 to be truthy + ✓ oma::compositor::covered_pixels(layer, src, 1000, 1000).has_value() to be falsy + validates render graphs + ✓ oma::compositor::validate(g, 1).has_value() to be truthy + ✓ oma::compositor::validate(g, 0).has_value() to be falsy + ✓ oma::compositor::validate(g, 1).has_value() to be falsy + ✓ oma::compositor::validate(g, 1).has_value() to be falsy + ✓ !r && r.error().code() == ErrorCode::InvalidArgument to be truthy + ✓ oma::compositor::validate(g, 1).has_value() to be falsy + +● compositor color + converts limited, full and 10-bit YUV + ✓ rgb_near(rgb(m8, 16.0 / 255, 128.0 / 255, 128.0 / 255), 0, 0, 0) to be truthy + ✓ rgb_near(rgb(m8, 235.0 / 255, 128.0 / 255, 128.0 / 255), 1, 1, 1) to be truthy + ✓ rgb_near(rgb(m8, 63.0 / 255, 102.0 / 255, 240.0 / 255), 1, 0, 0, 0.01) to be truthy + ✓ rgb_near(rgb(m10, 64.0 / 1023, 512.0 / 1023, 512.0 / 1023), 0, 0, 0) to be truthy + ✓ rgb_near(rgb(m10, 940.0 / 1023, 512.0 / 1023, 512.0 / 1023), 1, 1, 1) to be truthy + ✓ rgb_near(rgb(full, 1.0, 128.0 / 255, 128.0 / 255), 1, 1, 1) to be truthy + ✓ near(sd.matrix.at(0, 2), hd.matrix.at(0, 2), 1e-3) to be falsy + converts primaries and decodes transfers + ✓ near(same.at(0, 0), 1, 1e-9) && near(same.at(0, 1), 0, 1e-9) && near(same.at(2, 2), 1, 1e-9) to be truthy + ✓ near(m.at(0, 0), 1.6605, 1e-3) && near(m.at(0, 1), -0.5876, 1e-3) && near(m.at(0, 2), -0.0728, 1e-3) to be truthy + ✓ near(m.at(1, 0), -0.1246, 1e-3) && near(m.at(1, 1), 1.1329, 1e-3) to be truthy + ✓ near(m.at(2, 1), -0.1006, 1e-3) && near(m.at(2, 2), 1.1187, 1e-3) to be truthy + ✓ near(oma::compositor::to_linear(0.5, Transfer::Srgb), 0.214041, 1e-6) to be truthy + ✓ near(oma::compositor::to_linear(0.5, Transfer::Bt1886), std::pow(0.5, 2.4)) to be truthy + ✓ near(oma::compositor::to_linear(1.7, Transfer::Linear), 1.0) to be truthy + ✓ oma::compositor::resolve_transfer(16, &hdr) == Transfer::Bt1886 && hdr to be truthy + ✓ oma::compositor::resolve_transfer(13, &hdr) == Transfer::Srgb && !hdr to be truthy + +● compositor::CpuCompositor + fills the background premultiplied + ✓ out.has_value() to be truthy + ✓ static_cast(px[0]) to be close to 0.5 (precision 1e-06) + ✓ static_cast(px[1]) to be close to 0.25 (precision 1e-06) + ✓ static_cast(px[3]) to be close to 0.5 (precision 1e-06) + draws opaque layers and moves them by exact pixels + ✓ src.frame.has_value() to be truthy + ✓ a && b to be truthy + ✓ static_cast(a->at(0, 0)[3]) to be close to 1 (precision 1e-06) + ✓ shifted to be truthy + ✓ static_cast(b->at(5, 100)[1]) to be close to 0 (precision 1e-06) + ✓ static_cast(b->at(5, 100)[2]) to be close to 1 (precision 1e-06) + applies opacity and blend modes + ✓ full && half && doubled to be truthy + ✓ static_cast(h[0]) to be close to 0.0250907 (precision 1e-06) + ✓ static_cast(h[3]) to be close to 0.5 (precision 1e-06) + ✓ static_cast(d[1]) to be close to 1.93159 (precision 1e-06) + ✓ static_cast(d[3]) to be close to 1 (precision 1e-06) + rejects missing inputs + ✓ CpuCompositor{}.render(g, empty).has_value() to be falsy + ✓ CpuCompositor{}.render(g, {}).has_value() to be falsy + +● compositor::VulkanCompositor + matches the CPU reference on uploaded frames + ✓ vk.has_value() to be truthy + ✓ rendered.has_value() to be truthy + ✓ cpu && gpu to be truthy + max difference 0.00178, 0 of 230400 pixels over tolerance + ✓ diff.over_tolerance <= cpu->pixels.size() / 4 / 500 to be truthy + matches the CPU reference on zero-copy GPU frames + ✓ (*vk)->render(g, gpu_inputs).has_value() to be truthy + ✓ (*vk)->render(g, gpu_inputs).has_value() to be truthy + ✓ cpu && gpu to be truthy + VA-API to Vulkan frames: max difference 0.00121, 0 pixels over tolerance + ✓ diff.over_tolerance <= cpu->pixels.size() / 4 / 500 to be truthy + measures three 1080p layers + ✓ (*vk)->render(g, inputs).has_value() to be truthy + 1080p, 3 layers (GPU inputs): 3.93 ms per frame, submit to completion + +Test Suites Summary: + All 68 tests passed! + diff --git a/Docs/Research/hardware-strategy.md b/Docs/Research/hardware-strategy.md index 62fd7bc..50a9fa3 100644 --- a/Docs/Research/hardware-strategy.md +++ b/Docs/Research/hardware-strategy.md @@ -1,129 +1,83 @@ -# Hardware strategy: synthesis - -> Synthesis of [`vulkan.md`](vulkan.md), [`cuda.md`](cuda.md), [`opencl.md`](opencl.md) (OpenCL -> removed) and [`omarchy-integration.md`](omarchy-integration.md). It is a proposal for -> discussion: it becomes a rule only after it enters `CLAUDE.md` or an ADR. - -Goal: **use all available hardware** (decode, encode and compute on Intel, AMD and NVIDIA) -**without taking frames out of VRAM** and without requiring users to install anything beyond -what Omarchy already installs. - ---- - -## 1. Proposed architecture - -``` - ┌──────────────── libs/gpu ─────────────────┐ - │ single VkInstance / VkDevice (Vulkan-Hpp) │ - │ queue lock · VMA · imported memory │ - └───────┬──────────────┬──────────────┬──────┘ - │ │ │ - ┌────────────────────────▼──┐ ┌───────▼────────┐ ┌──▼──────────────────┐ - │ libs/media (FFmpeg 9) │ │ libs/compositor│ │ apps/omamovie (Qt) │ - │ hwcontext_vulkan on │ │ render graph → │ │ QQuickGraphicsDevice │ - │ OmaMovie's device │ │ Vulkan (+ libplacebo?)│ ::fromDeviceObjects│ - │ │ │ ComputeBackend │ │ QSGVulkanTexture │ - │ Decode: │ │ ├ Vulkan (default)│ ::fromNative │ - │ Intel/AMD: Vulkan Video │ │ ├ CPU (reference) │ - │ or VA-API→map │ │ └ CUDA (optional, PTX) │ - │ NVIDIA: NVDEC→interop │ │ │ - │ or Vulkan Video │ └────────────────┘ └─────────────────────┘ - │ Encode: │ - │ Intel/AMD: Vulkan/VA-API │ AVVkFrame (one timeline semaphore per image) - │ NVIDIA: NVENC │ = OmaMovie's Frame, without copies - │ Intermediates: ProRes/FFv1 │ - │ in compute shaders │ - └────────────────────────────┘ -``` - ---- - -## 2. Role of each API - -| API | Role | Required? | +# Hardware strategy after skeptical review + +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. + +## 1. Supported direction + +Use the existing Vulkan compositor with explicit resource ownership, bounded work and a +logged software-decode fallback. Preserve the ADR-0004 application-owned device design, +while keeping Qt integration provisional until Q1/S4. No vendor-wide real-time or zero-copy +promise follows from one Intel report. + +## 2. Roles and alternatives + +| Component | Current direction | Evidence boundary / alternative | +|---|---|---| +| Vulkan | Composition, effects, optional video/compute codecs | Query device/profile/features and verify output | +| VA-API | First decode candidate on the reference Intel setup | Import may fail; software + upload remains baseline | +| Vulkan Video | Candidate when normally exposed | Debug-flag tests are exploratory, not supported defaults | +| NVDEC/NVENC | NVIDIA candidates | S7 pending; compare native Vulkan and total interop cost | +| CUDA effects | Optional specialization | Need an actual effect and same-output measurement | +| OpenCL | Excluded by project scope | Generic interop exists; no current use case | +| libplacebo | Color/scaling candidate | S6 and ADR-0006 pending | +| VMA | Allocation candidate | Existing wrappers first; adopt on demonstrated need | + +[API/source evidence](sources.md) establishes contracts, not total pipeline performance. + +## 3. Runtime policy + +Distinguish five states: available API/library, advertised capability, successful open, +actual negotiated hardware frame, and validated path for this workload. Only the last +supports a compatibility/performance claim. Log skipped/failed paths and reasons. + +ADR-0004's VA-API → Vulkan → software order is provisional for the measured Intel setup. +Do not infer NVDEC availability from `libnvcuvid` presence. Match device identity for imports; +probe support at codec/profile/chroma/depth/size granularity. Driver-specific policy changes +need evidence, not a universal vendor ranking. Software fallback is correct behavior and +still needs a workload budget; it is not guaranteed real time. + +## 4. Device and synchronization choices + +A shared device reduces explicit inter-device imports. A device created by FFmpeg could also +be handed to a consumer; it does not inherently force a second device or copy. The reason +for OmaMovie ownership is control over creation/features/lifetime (current ADR), not that +other ownership is impossible. + +Queues need host synchronization; images need GPU dependencies, layouts and lifetime. +Qt's ordinary imported-device path has an identified flagged-queue mismatch (Q1). +An application mutex protects Qt only if Qt submissions actually participate in that +protocol. Decide among render-thread submission, controlled rendering, verified native +queue injection, or separate-device external-memory import in S4/ADR-0005. + +## 5. Experiments and gates + +| Spike | Review status | Required additional evidence | |---|---|---| -| **Vulkan (Vulkan-Hpp)** | Single device; compositing; preview; default compute for effects; video decode/encode on AMD/Intel; compute codecs (ProRes, FFv1) | **Yes** | -| **VA-API** | Mature decode/encode on Intel/AMD; mapped to Vulkan by FFmpeg | Yes (through FFmpeg), chosen at runtime | -| **NVDEC/NVENC** | Decode/encode on NVIDIA; NVENC is much faster than Vulkan encode | Yes on NVIDIA (through FFmpeg, no toolkit) | -| **CUDA (driver API)** | NVDEC → Vulkan interop; optional kernels | Loaded at runtime, NVIDIA only | -| **CUDA (kernels)** | Effects with a measured gain on NVIDIA | Optional, optional build | -| ~~OpenCL~~ | **Removed** (decision of 2026-10-02, see `opencl.md`) | No | -| **libplacebo** | Candidate for color, tone mapping, scaling, LUTs | To be decided (ADR) | - ---- - -## 3. Runtime selection - -At startup, `libs/gpu` + `libs/media` build a **capability table** (logged in the `gpu` -category and shown on a diagnostics screen): - -| Question | Source | -|---|---| -| Which codecs/profiles/bit depths does Vulkan Video decode and encode? | `vkGetPhysicalDeviceVideoCapabilitiesKHR` | -| What does VA-API offer? | `vaQueryConfigProfiles` / FFmpeg | -| Is NVDEC/NVENC present? | Presence of `libnvcuvid`/`libnvidia-encode`, FFmpeg | -| Is CUDA present? | `dlopen` of `libcuda` | -| Which GPU is primary (hybrids)? | User choice or a heuristic, without waking the dGPU needlessly | - -For each stream, the decode path is chosen by an **ordered, testable policy** (e.g. Intel H.264 -→ VA-API if the benchmark favors it, else Vulkan Video → software). A software fallback is -always possible, and every degradation is logged (`CLAUDE.md` §19). - ---- - -## 4. Changes relative to the current `CLAUDE.md` - -Proposals (pending the maintainer's approval): - -1. ~~OpenCL as the generic backend~~ **Applied on 2026-10-02**: OpenCL removed; Vulkan Compute - is the generic backend (`CLAUDE.md` §4 and §9.3). -2. **Explicit Vulkan-Hpp** in the stack: `vk::raii`, no exceptions (`VULKAN_HPP_NO_EXCEPTIONS` + - `VULKAN_HPP_RAII_NO_EXCEPTIONS` with `std::expected`), include confined to - `libs/gpu`/`libs/compositor`. Affects §4 and §22. -3. **A single Vulkan device shared with FFmpeg and Qt** as an invariant. Affects §7 and §9.4. -4. **CUDA policy**: NVDEC/NVENC through FFmpeg without the toolkit; kernels only through PTX - (clang) + `dlopen`; optional CUDA build. Affects §4 and §9.3. -5. **Hybrid laptops**: one primary device per pipeline. Affects §7. -6. **Omarchy integration**: app_id `omamovie`, opacity rule, theme through - `omarchy-theme-color`, an isolated platform module. Affects §11. -7. **The `vulkan-headers` package** (and `vulkan-tools` for diagnostics) in the build dependencies. - ---- - -## 5. Validation spikes (before the definitive code) - -Each spike has a measurable criterion. They replace the technical part of the "first -experiment" in `CLAUDE.md` §24. - -| # | Spike | Success criterion | Hardware | -|---|---|---|---| -| S1 | `vulkaninfo` + `vainfo`: capability inventory on the Iris Xe | Documented codec × API table | Iris Xe | -| S2 | FFmpeg 9 using a `VkDevice` created by OmaMovie (Vulkan-Hpp) | H.264/HEVC/AV1 decoded into `AVVkFrame` without readback | Iris Xe | -| S3 | Vulkan Video vs. VA-API→map: frame time, CPU, power | Numbers per codec; selection policy defined | Iris Xe | -| S4 | Qt Quick on the same device showing the compositor image | 60 fps preview without a CPU copy; queue lock without deadlock | Iris Xe | -| S5 | Compositing 2 videos + 1 image (transform, crop, opacity, YUV→RGB) | Frame time measured at 1080p60 | Iris Xe | -| S6 | libplacebo on OmaMovie's images | Color conversion/scaling without copies; decision for the ADR | Iris Xe | -| S7 | NVDEC → Vulkan (interop) vs. Vulkan Video on NVIDIA | NVIDIA path chosen by measurement | **Needs an NVIDIA machine** | -| S8 | AMD (RADV) | Same measurements as S3 | **Needs an AMD machine** | - ---- - -## 6. Main risks - -| Risk | Mitigation | -|---|---| -| Queue synchronization between Qt, FFmpeg and the compositor | S4 early; a single lock mechanism; ADR-0005 | -| Driver differences in DMA-BUF/modifiers | Use FFmpeg's mapping; fallback with a logged GPU→GPU copy | -| CUDA does not accept wait-before-signal | The scheduler guarantees the order; specific tests | -| No AMD/NVIDIA hardware for testing | Plan test machines or contributors; CI only covers software paths | -| Vulkan-Hpp compile cost | Confined include + PCH; evaluate the `vulkan` module | -| libplacebo not accepting external images as expected | S6 before deciding | - ---- - -## Suggested next steps - -1. The maintainer decides the remaining points of §4 (mainly libplacebo). -2. Update `CLAUDE.md` with whatever is approved. -3. Install `vulkan-headers vulkan-tools libva-utils` and run S1. -4. Continue with S2–S4 (M1 in `Docs/implementation-plan.md`). +| S1 | Prior Intel report, not rerun | Retain capability vs executed-codec distinction; raw output/provenance | +| S2 | Prior single-submitter report, not rerun | Validation layers, chroma/multiple frames, lifetime/cancellation | +| S3 | Pending | Repeated equivalent workloads; decode + import + consumer, encode quality | +| S4 | Pending; Q1 identified | Queue retrieval, submission/lifetime protocol, display conversion | +| S5 | Library tests contain timings | End-to-end playback, percentiles/drops, independent color oracle | +| S6 | Pending | Installed libplacebo contracts, image-quality and copy comparisons | +| S7 | Pending; NVIDIA unavailable | Decode+interop+composition and export on actual NVIDIA | +| S8 | Pending; AMD unavailable | Same criteria on actual AMD | + +Protocols and stop criteria live in [the audit](skeptical-review.md#validation-protocols). +No pending experiment is relabeled as completed by this documentary review. + +## 6. Capacity model + +Use measured resource budgets, not only throughput. Tightly packed 4K RGBA16F is 63.28 MiB +per frame; three streams with eight such cached frames consume about 1.48 GiB before decoder +pools, intermediate targets and padding. This is a calculated lower-bound illustration, +not measured VRAM usage. [Inputs and calculations](skeptical-review.md#quantitative-checks). +On integrated GPUs, GPU-local allocation does not imply physically separate VRAM. + +## 7. Application decision + +Continue bounded Intel correctness work. Resolve Q1 and SDR/display handling before wiring +the final UI. Support claims must name tested hardware/drivers/media; untested AMD/NVIDIA +remain planned. Choosing a proxy codec also needs storage, seek, quality and compatibility +measurements; GPU compute decode alone does not justify FFv1/ProRes as the default. diff --git a/Docs/Research/imovie.md b/Docs/Research/imovie.md index 7cecdda..ccf3b2a 100644 --- a/Docs/Research/imovie.md +++ b/Docs/Research/imovie.md @@ -1,229 +1,54 @@ -# Apple iMovie: UI/UX +# iMovie: documented interactions and UX hypotheses -> Research done on 2026-10-02. Sourced facts are linked in the [Sources](#sources) section. -> Items marked **(general observation)** come from broad knowledge of the product without a -> specific source in this research; verify before using them as requirements. -> -> The goal is to extract UX **principles**. `CLAUDE.md` forbids copying iMovie's visual identity -> or proprietary behavior. +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. ---- +## 1. Conclusion -## 1. Summary +Contextual adjustment controls, editable named overlays and skimming are useful references. +They support prototype choices, not a finding that OmaMovie's proposed UI is already easy. +The claim that iMovie shares Final Cut's internal engine is withdrawn from architectural +justification: the inspected Apple guide establishes transfer, not implementation identity. -iMovie is simple because: +## 2. Documented behavior -1. the screen has **three fixed areas** (browser, viewer, timeline) and little else; -2. adjustment controls appear **in context**, in a bar above the viewer; -3. compositing effects are offered as **intent-named operations** (picture-in-picture, - cutaway, split screen, green screen), not as generic layers and parameters; -4. the timeline is **magnetic**: it leaves no gaps and avoids collisions; -5. there are **templates** (themes, trailers, storyboards, Magic Movie) for those who do not want to start from scratch; -6. underneath, since 2013, it uses the **Final Cut Pro X engine**, and a project can be sent to Final Cut Pro. +[Apple's Mac guide](https://support.apple.com/en-nz/guide/imovie/welcome/mac) lists trimming, +clip color, skimming, ratings, multiple-clip effects and conversion of trailers to movies. +Its examples put adjustments near the viewer. These are documented interactions; their +presence does not quantify discoverability or learnability for Omarchy users. -Item 6 is the most relevant for OmaMovie: it is exactly "simple UI, serious pipeline". +[Sending to Final Cut](https://support.apple.com/guide/imovie/send-projects-to-final-cut-pro-movcbf7e2a3f/mac) +copies project/media and maps at least one audio adjustment to a Gain filter. Trailers must +first be converted. This proves a supported workflow with conditions, not shared code, +unlimited capability, or lossless mapping of every effect. ---- +## 3. Foundations and model -## 2. History relevant to UX +Separate presentation from persisted edit semantics. A contextual drawer changes visible +controls; it must not become the owner of clip parameters. Named operations may emit normal +model commands; that is an OmaMovie design inference, not a description of Apple's internals. -| Year | Version | Change | -|---|---|---| -| 1999 | iMovie 1 | Launched with the iMac DV | -| 2004 | iMovie 4 | Non-destructive editing ("Direct Trimming") | -| 2005 | iMovie HD (5/6) | HDV support | -| 2007 | **iMovie '08** | Full rewrite by Randy Ubillos focused on speed. Introduced **skimming** and **Events** (clips grouped by day). **Heavily criticized**: removed timeline audio and plugins. David Pogue called it "an utter bafflement" | -| 2009–2010 | iMovie '09 / '11 | Brought removed features back; added **trailers** and the **Precision Editor** | -| 2013 | **iMovie 10** | New rewrite on the **Final Cut Pro X engine**; libraries | -| 2015 | 10.1 | 4K editing | -| 2020 | 10.2 | Apple silicon | -| 2022 | iOS 3.0 | **Storyboards** and **Magic Movie** | +Magnetism/ripple is a chosen editing policy over generic tracks. Specify linked audio, +connected overlays, gaps, locked tracks and undo transaction boundaries before implementing +it. A single simple-looking sequence can hide surprising moves on other tracks. -### The iMovie '08 lesson +## 4. Alternatives and proposed test -The rewrite brought good ideas (skimming, organization by events) but removed capabilities -users already had. The negative reaction forced Apple to put features back over the next two -versions. +Compare contextual drawer with a fixed side inspector; compare ripple default with ordinary +track placement. M6 tasks: import, split, trim, overlay, gain and undo at wide/half/narrow +sizes using mouse and keyboard. Record completion, mistakes, recovery, hidden-control +searches and surprise track movements. Prefer a design only after representative tasks +show no required action is inaccessible and improvement does not break precise editing. +No usability test was executed by this review. -**For OmaMovie:** simplifying the UI **must never mean removing capability** already shipped. -That is the "hide, do not remove" principle of `CLAUDE.md` (§11), with a concrete historical case. +Skimming requires cancelable decode/cache work, but every hover need not trigger a full seek. +Compare cached thumbnails with live decode; track request age and stale output, not just FPS. ---- +## 5. Removed arguments and application limits -## 3. Window layout - -According to the official guide, the window has three areas: - -- **Browser**: media and content library (transitions, titles, audio, backgrounds). -- **Viewer**: preview, with filters and effects applied. -- **Timeline**: the project sequence, video on top and audio below. - -Interaction details: - -- **A single divider resizes everything**: dragging the border between viewer and timeline up - or down adjusts all three areas proportionally. -- The **Libraries List** and the **browser** can be hidden (the "Content Library" toolbar button). -- **Window > Revert to Original Layout** restores the default layout. -- There are no free panels or workspaces. - -### Implications for OmaMovie - -- The target layout in `CLAUDE.md` (MediaPanel, PreviewPanel, TimelineView, Inspector, - TransportControls) is similar in spirit. To keep it simple: **few dividers, a fixed layout, - "restore layout" always available**. -- In Hyprland the user already manages windows in a tiling layout. A single window with a fixed - internal layout fits that better than floating panels. - ---- - -## 4. Contextual adjustments bar - -- The **adjustments bar** sits above the viewer. When a clip is selected in the browser or the - timeline, its buttons show the corresponding controls. -- Documented example (color): the **Color Balance** button and the **Color Correction** - button, which shows a multi-slider control, saturation and color temperature. -- Applied state is visible: buttons are highlighted when an automatic adjustment is applied to the clip. -- Interactive tools (e.g. the eyedropper) appear **in the viewer** when needed. -- The guide lists other adjustments of the same kind: auto enhance, crop, rotation, Ken Burns, - stabilization, filters, speed, volume. - -### Implications for OmaMovie - -- This is the **contextual Inspector** model of `CLAUDE.md` §11, with one detail to adopt: - **visually show which adjustments are active on the clip** (badge or highlight) without - opening each section. -- **Direct manipulation in the preview** (crop, position, eyedropper) beats numeric fields for - most users. Numeric values should exist in the inspector for precision. Technical implication: - the PreviewPanel needs interactive overlays that talk to the model through commands (with - undo), without keeping state of their own. - ---- - -## 5. Timeline and editing - -### Magnetic timeline - -The concept comes from Final Cut Pro X, whose engine iMovie uses: clips "magnetically" adjust -around a dragged clip, and when a clip is removed its neighbors close the gap. There are no -accidental gaps. - -### Trimming methods (documented) - -1. **Drag the clip edges** in the timeline. -2. **Clip Trimmer**: shows the unused part of the clip dimmed; lets you extend, shorten or slide - the used section without changing duration (equivalent to a **slip**). -3. **Precision Editor**: double click an edge; adjusts the start and end of clips and the - duration of transitions; supports **split edits** (audio and video with different cut - points, J-cut/L-cut). -4. **Range selection** (hold R + drag) and "Trim Selection". - -### Overlays as named operations - -Picture-in-picture, cutaway, split screen and green screen are offered as **high-level -effects**: the user places a clip above another and picks the intent. No need to understand -layers, masks or blend modes. Compositing capability is **limited** (no arbitrary multiple -video layers and no general keyframes) **(general observation)**. - -### Implications for OmaMovie - -- **Magnetic timeline as the default UI behavior**, implemented as ripple in the model. The - timeline model stays generic (free tracks, gaps allowed); "magnetism" is an editing policy - applied by the UI's commands. -- **Precision Editor = a friendly UI for roll and split edits.** It shows advanced trimming - can be simple with good visualization (showing the unused material). The command model in - `premiere-pro.md` §3 serves both audiences. -- **Named operations compiled to the generic compositor**: "Picture in picture" in the UI - becomes layers + transform + crop + border in the render graph. OmaMovie can offer iMovie's - presets **without** its layer limitation, because the compositor is generic. This applies - "simple UI, serious pipeline" directly. -- The preset must stay editable: after applying "PiP", the inspector shows normal - transform/crop, and the user can go beyond what the preset provided. - ---- - -## 6. Library and skimming - -- **Skimming** (since iMovie '08): hovering over a clip shows its content at a user-controlled - speed, without pressing play. -- Thumbnail size is adjustable with a slider, as is how much time each thumbnail represents and whether waveforms show. -- Clips can be **rated** (favorite/rejected) and filtered. -- Media is organized into **libraries** and **events**. - -### Implications for OmaMovie - -- Skimming is the interaction that most sets a "visual" library apart. **It needs - architecture**: fast decoding of arbitrary frames (seek), a thumbnail cache at several - densities, and work off the UI thread with aggressive cancellation (the mouse moves faster - than decode). This feeds the cache (§15) and job system (§13) requirements of `CLAUDE.md`. -- Clip ratings (favorite/rejected) are light metadata. A good case for SQLite (§17). - ---- - -## 7. Templates and automation - -- **Themes**: a visual style applied to the titles and transitions of the whole movie. -- **Trailers**: Apple templates per genre, with a shot list and titles; they can be converted into a normal project. -- **Storyboards** (iOS 3.0): ~20 suggested outlines (recipe, Q&A, product review, news report), - with tips per scene; reorderable and editable. -- **Magic Movie** (iOS 3.0): builds a video from selected clips, identifying dialogue, faces - and actions, and adds transitions, music and titles. - -### Implications for OmaMovie - -- The most important point: **a trailer can be converted into a normal project**. Templates - must produce ordinary timeline structures, not a closed special mode. In OmaMovie a template - would be a generator of commands/ProjectIR, without a separate project type. -- Templates and automatic generation are not a priority (initial non-goals), but the "template - → normal timeline" design costs little if planned. - ---- - -## 8. A path to professional tools - -- iMovie has used the Final Cut Pro X engine since 2013. -- The guide documents **sending projects to Final Cut Pro**. - -### Implications for OmaMovie - -- Validates the project's thesis: a simple UI on a professional engine, with no dead end when - the user grows. -- Difference for OmaMovie: Apple has two apps; OmaMovie wants **one app** in which advanced - capability is hidden, not moved into another product. That requires more discipline in the - contextual inspector and in progressive disclosure. - ---- - -## 9. Shortcuts - -The guide has a keyboard shortcuts page (window navigation, editing, playback). iMovie is -usable from the keyboard but mouse-oriented **(general observation)**. - -**For OmaMovie:** `CLAUDE.md` requires full keyboard editing, which means going beyond iMovie -here, closer to Premiere. - ---- - -## 10. What not to copy - -- Visual identity, icons, brand names ("Magic Movie", "Precision Editor") and trade dress. -- **Capability limits** that exist only to keep things simple (fixed number of layers, no - keyframes). OmaMovie hides complexity; it does not remove it. -- Ecosystem dependence (Photos, iCloud). - ---- - -## Sources - -- [Apple: iMovie User Guide for Mac](https://support.apple.com/guide/imovie/welcome/mac) -- [Apple: Change the window layout in iMovie on Mac](https://support.apple.com/guide/imovie/change-the-window-layout-movcab8cb6ef/mac) -- [Apple: Trim clips in iMovie on Mac](https://support.apple.com/guide/imovie/trim-clips-movf8b8fc9b2/mac) -- [Apple: Adjust a clip's color in iMovie on Mac](https://support.apple.com/guide/imovie/adjust-a-clips-color-movf2a3f0f42/mac) -- [Apple: Keyboard shortcuts in iMovie on Mac](https://support.apple.com/guide/imovie/keyboard-shortcuts-movd9d8f91e8/mac) -- [Apple: Send projects to Final Cut Pro](https://support.apple.com/guide/imovie/send-projects-to-final-cut-pro-movcbf7e2a3f/mac) -- [Apple: Final Cut Pro interface (Magnetic Timeline, browser, viewer)](https://support.apple.com/en-vn/guide/final-cut-pro/ver92bd100a/mac) -- [Apple: Intro to the Magnetic Timeline in Final Cut Pro for iPad](https://support.apple.com/it-it/guide/final-cut-pro-ipad/devd9b6a5715/ipados) -- [Wikipedia: iMovie](https://en.wikipedia.org/wiki/IMovie) -- [Engadget: Maybe iMovie '08 isn't such a bad change after all](https://www.engadget.com/2007-08-27-maybe-imovie-08-isnt-such-a-bad-change-after-all.html) -- [Joseph Dickerson: iMovie 08 – a step back?](https://www.josephdickerson.com/blog/2007/08/19/imovie-08-a-step-back/) -- [XDA: Apple iMovie 3.0 introduces Storyboards and Magic Movie](https://www.xda-developers.com/apple-imovie-3-features/) -- [Videomaker: iMovie 3.0 receives two major time-saving features](https://www.videomaker.com/news/imovie-3-0-receives-two-major-time-saving-features/) +The old iMovie '08 history/reaction narrative was not checked against contemporary primary +records and no longer proves a universal prohibition on removing features. Preservation of +shipped capability is a product preference; removal may need explicit migration/deprecation. +No Mac/iOS feature list is treated as interchangeable. Shortcut choices in `ui-design.md` +are OmaMovie assignments, not a certified translation of all Apple shortcuts. diff --git a/Docs/Research/movie-maker.md b/Docs/Research/movie-maker.md index 82a29ef..610ff1c 100644 --- a/Docs/Research/movie-maker.md +++ b/Docs/Research/movie-maker.md @@ -1,160 +1,53 @@ -# Microsoft Movie Maker (and successors): UI/UX +# Movie Maker and Clipchamp: bounded UI references -> Research done on 2026-10-02. Sourced facts are linked in the [Sources](#sources) section. -> Items marked **(general observation)** come from broad knowledge of the product without a -> specific source in this research. -> -> Covers Windows Movie Maker (2000–2009), Windows Live Movie Maker (2009–2017) and, as context, -> its successors: the Photos app Video Editor and Clipchamp. +> Reviewed on 2026-10-03 with skeptical-research. This is non-normative research. +> Source metadata and access limits: [source register](sources.md). Decisions and open +> validation gates: [audit](skeptical-review.md). Product descriptions are not user studies. ---- +## 1. Conclusion -## 1. Summary +A storyboard projection and contextual properties remain prototype candidates. The old +claim that Movie Maker 2012 zoom changes equal-width clips into duration-scaled rectangles +was not verified in a primary record. Do not implement semantic zoom on that historical +claim alone. -For many people Movie Maker was their first video editor. The useful lessons are: +## 2. Current documented baseline -1. **Storyboard and timeline as two views of the same project.** In the last version (2012) - they became one area with **zoom**: zoomed out it is a storyboard; zoomed in it is a timeline. -2. **A task-guided flow** (capture → edit → publish). -3. **AutoMovie**: predefined styles that assemble a movie automatically. -4. Negative lessons: **narrow format support** in the first version and **features removed** - between versions, the same mistake as iMovie '08. +[Microsoft's Clipchamp timeline guide](https://support.microsoft.com/en-us/clipchamp/how-to-work-with-the-timeline-in-clipchamp) +describes dragging library assets onto the timeline, rearranging, trimming, multiple assets +and properties such as audio. It establishes an interaction baseline, not causal evidence +that a similar arrangement will make OmaMovie usable. ---- +Library/viewer/timeline/properties is one way to make selection-dependent tools reachable. +The existing OmaMovie drawer remains a product choice; compare a side-panel alternative +rather than declaring a layout validated through visual resemblance. -## 2. History +## 3. Historical evidence boundary -| Date | Version | Notes | -|---|---|---| -| 2000-09-14 | 1.0 (Windows Me) | Criticized for its small feature set and for exporting only ASF | -| 2001 | 1.1 (Windows XP) | DV-AVI and WMV 8 export | -| 2002-11 | **2.0** | Large feature expansion; 60+ transitions and 37 effects; custom XML effects | -| 2004–2005 | 2.1 (XP SP2), MCE 2005 | DVD burning in Media Center Edition | -| 2007 | **6.0 (Vista)** | DirectX/Direct3D effects, HDV; **removed analog capture** and XML customization. Version 2.6 released for older hardware | -| 2009-08-19 | **Windows Live Movie Maker** | Rewrite: **ribbon** toolbar (Office 2007 style), **AutoMovie**, direct publishing to YouTube/Facebook. **Removed** stabilization and narration | -| 2010-08 | 2011 | Brought back HD video and webcam capture | -| 2012-04 | 2012 | H.264/MP4 as the default export; narration back | -| 2017-01-10 | — | **Discontinued** | -| Windows 10 | Video Editor (Photos app) | Replacement, with text-to-speech and OneDrive | -| 2021-09-08 | **Clipchamp** | Acquired by Microsoft; Windows 11's default editor; brings back the multitrack timeline | +The earlier version-by-version dates, format/export restrictions and removal/restoration +claims came mainly from secondary summaries. They were not needed to choose the next +engineering step and have been removed from active guidance. The attempted Microsoft +Windows Essentials URL was unavailable in this audit (source register). -In 2013 PCMag called it "possibly the simplest (and funnest) way to combine video clips". +Neither discontinuation nor a feature's disappearance establishes that architecture caused +product failure. Treat that causal story as unsupported, not a foundation for modularity. ---- +## 4. Storyboard hypothesis -## 3. Classic interface (versions 2.x–6.0) +An equal-width clip projection can help ordering but destroys the correspondence between +screen distance and timeline time. It needs explicit transition into duration-scaled mode, +a clearly mapped playhead, and gestures whose effects remain predictable. Free overlays +and split audio make a single-row storyboard especially ambiguous. -- **Task pane** on the left, organized in steps (capture video, edit movie, finish movie) - **(general observation; Wikipedia describes the layout in general terms)**. -- **Collections**: organization of imported media. -- Real-time **preview**. -- **Storyboard / Timeline** at the bottom, switchable: - - **Storyboard**: large fixed-size boxes for each clip or image, with smaller boxes between - them for transitions. Used to order the material. - - **Timeline**: each rectangle's length is proportional to its duration; supports trimming, - adjusting transition duration and shows the audio track. - - Separate tracks for video, audio/music and titles. +Compare conventional time zoom, a minimap, and semantic storyboard in M6/M8. Use identical +projects including unequal durations, multiple tracks, gaps and transitions. Accept only if +users can locate the same cut, reorder/undo without changing unintended tracks, and return +to precise time editing without losing context. Measure errors and navigation work; do not +assume fewer modes or fewer pixels is automatically simpler. Test is proposed, not run. -### What worked +## 5. Application -- A storyboard is great for the first assembly (scene order) and is less intimidating. -- A transition as a visible object **between** two clips, with its own box, makes the concept obvious. -- The step-by-step flow taught the editing process. - -### What did not work - -- Switching modes is a context switch: users must know which mode they are in to understand what they can do. -- The track set was fixed and limited. - ---- - -## 4. Windows Live Movie Maker (2009–2012) - -- A **ribbon** instead of menus, with tabs per category (Home, Animations, Visual Effects, - Project, View) **(general observation for the tab names)**. -- **A hybrid storyboard/timeline area with a zoom slider** in the bottom-right corner. With the - slider all the way left, each clip is a same-size rectangle (storyboard). Zooming in, the - rectangles start reflecting duration (timeline). -- **AutoMovie**: pick a style and the app builds the movie with title, credits, transitions and music. -- **Direct publishing** to platforms (YouTube, Facebook). -- Simplified editing: one main video track, no free multitrack **(general observation)**. - -### Implications for OmaMovie - -- **Semantic zoom** is the most reusable idea in this research. A single timeline component - with a zoom axis that goes from "storyboard" (one thumbnail per clip, uniform size, focus on - order) to "timeline" (proportional to time, focus on cuts). No separate mode and no context switch. - - Technical requirement: the TimelineView needs a layout that is not linearly scaled at the - zoomed-out extreme. The timeline model does not change; only the visual projection does. - - Needs measuring: thumbnails for every clip at minimum zoom require a cache and list virtualization. -- The ribbon is an Office pattern, not a video editor pattern. Do not adopt it; the contextual - inspector plays the same role with less fixed space. -- AutoMovie and templates: same conclusion as iMovie (`imovie.md` §7): produce a normal, editable timeline. - ---- - -## 5. Successors - -### Photos Video Editor (Windows 10) - -Replaced Movie Maker with an even simpler, storyboard-based approach, with text-to-speech and -OneDrive integration. Wikipedia notes that timeline-based editing only returned with Clipchamp, -so Windows went years without a built-in timeline editor. - -### Clipchamp (Windows 11) - -A three-column layout plus a timeline: - -- **Left sidebar with tabs**: Your media, Record & create (webcam, screen, text-to-speech), - Templates, Music & SFX, video/image libraries (stock), Graphics, Text, Transitions, Brand kit. -- **Preview** in the center. -- **Multitrack timeline** at the bottom. -- **Contextual properties panel on the right**: changes with the selected element, with tabs - for color, audio, filters, speed, transitions and text. -- Templates by target format (Instagram, TikTok, etc.) and AI-generated videos. -- Export at several resolutions (4K on the paid plan) and direct upload to social media. -- Reviews describe the interface as "refreshingly simple" and similar to Premiere in arrangement. - -### Implications for OmaMovie - -- Clipchamp arrived at the same layout OmaMovie plans: library on the left, preview in the - center, contextual inspector on the right, timeline at the bottom. That convergence validates - the design in `CLAUDE.md` §11. -- **A library with tabs per content type** (media, text, transitions, audio) is an alternative - to iMovie's "single browser". For OmaMovie, the version without stock content: - Media / Titles / Transitions / Effects. -- **Integrated screen/webcam recording** is a feature with real demand in simple editors. On - Omarchy/Wayland it would go through PipeWire and xdg-desktop-portal (ScreenCast). Not a - priority; record it as a future idea. -- Technical contrast: Clipchamp is built on web technology **(general observation)**. OmaMovie, - native and GPU-first, should compete on performance and on working offline without an account. - ---- - -## 6. Negative lessons - -| Mistake | Version | Lesson for OmaMovie | -|---|---|---| -| Exported only ASF | 1.0 | Broad format support is part of usability. FFmpeg solves it; do not restrict artificially | -| Removed analog capture | 6.0 | A platform change does not justify losing capability | -| Removed stabilization and narration | Live 2009 | A rewrite that removes features causes rejection (same as iMovie '08) | -| Removed effect customization | 6.0 and Live | Withdrawn extensibility does not come back. Do not promise extensibility (plugins) before it can be maintained, as `CLAUDE.md` already says | -| Discontinued without an equivalent successor | 2017 | Users spent years without a timeline on Windows. An architecture that needs a full rewrite tends to kill the product; hence the "grow without rewriting the core" rule | - -An internal memo from Bill Gates (January 2003), revealed in an antitrust case, complained about -how hard it was to download and install Movie Maker. **Lesson:** the first experience (install, -open, import) is part of the product. For OmaMovie: simple packaging for Arch/Omarchy and a -first run with no configuration. - ---- - -## Sources - -- [Wikipedia: Windows Movie Maker](https://en.wikipedia.org/wiki/Windows_Movie_Maker) -- [InformIT: Using Microsoft Movie Maker 2](https://informit.com/articles/article.aspx?p=370630) -- [USF Tech Ease: What is Windows Movie Maker](https://etc.usf.edu/techease/win/images/what-is-windows-movie-maker/) -- [O'Reilly: Customizing your Timeline view (Windows Movie Maker 2)](https://www.oreilly.com/library/view/microsoft-windows-movie/0321199545/0321199545_ch09lev1sec5.html) -- [XDA: How to use the Clipchamp video editor on Windows 11](https://www.xda-developers.com/how-to-use-clipchamp-video-editor-windows-11/) -- [ITPro: Clipchamp to be installed on Windows 11 by default](https://itpro.com/operating-systems/microsoft-windows/366289/clipchamp-video-editor-to-be-pre-installed-on-windows-11) -- [Help Desk Geek: How to use the Windows 11 video editor (Clipchamp)](https://helpdeskgeek.com/how-to-use-the-windows-11-video-editor-clipchamp/) -- [Windows Report: Clipchamp review](https://windowsreport.com/clipchamp-review/) +Keep storyboard as an M8 evaluation item. Keep broad media support, installation usability +and recoverable projects for their direct product value, independent of unverified history. +Prices, account/plan restrictions and browser performance were not measured and do not +support an OmaMovie performance advantage. diff --git a/Docs/Research/omarchy-integration.md b/Docs/Research/omarchy-integration.md index 569cb65..0eb3a78 100644 --- a/Docs/Research/omarchy-integration.md +++ b/Docs/Research/omarchy-integration.md @@ -1,212 +1,80 @@ -# Omarchy integration - -> Research done on 2026-10-02 by **inspecting the local Omarchy 4.0.4 installation** -> (`/usr/share/omarchy`, `~/.config/omarchy`, `~/.local/state/omarchy`). Paths and formats may -> change between Omarchy versions. Every integration must have a fallback and live in a single -> app module (`CLAUDE.md` §11). - ---- - -## 1. Summary - -Omarchy 4 offers concrete integration points that are stable enough for OmaMovie to use: - -| Point | Mechanism | Use in OmaMovie | -|---|---|---| -| **Theme** | `~/.local/state/omarchy/current/theme/colors.toml` + `omarchy-theme-color` | UI colors, live switching | -| **Window opacity** | Hyprland rules in Lua | **A 100% opaque window** (correct preview color) | -| **Screen recordings** | `gpu-screen-recorder` → `$XDG_VIDEOS_DIR` | A "Recordings" source in the library | -| **Shell** (Quickshell/QML) | Plugins with `manifest.json`, IPC | Export progress widget (optional) | -| **Omarchy menu** | `~/.config/omarchy/extensions/omarchy-menu.jsonc` | Shortcuts: new project, edit last recording | -| **Notifications** | `omarchy-notification-send` / D-Bus | "Export finished" | -| **Hooks** | `~/.config/omarchy/hooks/.d/` | React to theme and font changes | -| **Drivers** | Omarchy's per-vendor installer | Know which hardware path to expect | - -Product finding: **Kdenlive is installed by default on Omarchy** (`install/omarchy-base.packages`), -together with OBS, mpv, `gpu-screen-recorder` and `ffmpegthumbnailer`. OmaMovie competes for -that default editor slot. - ---- - -## 2. Omarchy 4 technical context - -- Arch Linux + Hyprland; **the Hyprland configuration is written in Lua** (`default/hypr/*.lua`). -- **`omarchy-shell`**: a single **Quickshell** (QML) instance that hosts the bar, menus, panels, - notifications and overlays as plugins. **It is the same UI technology as OmaMovie (Qt Quick/QML).** -- More than 440 `omarchy-*` commands in `/usr/share/omarchy/bin`, with metadata (`omarchy:summary`, `omarchy:args`). -- Installed as a package (`omarchy 4.0.4-1`), with its own kernel (`linux-omarchy`). - ---- - -## 3. Theme - -### Format -Each theme's `colors.toml` defines: -```toml -mode = "dark" -accent = "#89b4fa" -selection = "#45475a" -muted = "#585b70" -background = "#1e1e2e" # + dark_background, darker_background, lighter_background -foreground = "#cdd6f4" # + dark_foreground, light_foreground, bright_foreground -red/yellow/orange/green/cyan/blue/magenta/brown (+ bright_* variants) -``` - -### How a theme is applied (`omarchy-theme-set`) -1. Assembles the theme in `~/.local/state/omarchy/current/next-theme` (official theme + user overlay). -2. Generates configs from `*.tpl` templates (`{{ accent }}`, `{{ accent_rgb }}`, `{{ mix a b 30% }}`...). -3. **Atomic swap**: `mv next-theme → theme`, writes `theme.name`. -4. Sends the palette to the shell over IPC, restarts/retints known apps (`omarchy-theme-set-obsidian`, `-vscode`...). -5. Runs `omarchy-hook theme-set ` (scripts in `~/.config/omarchy/hooks/theme-set.d/`). - -### Color resolver -`omarchy-theme-color --all` prints `keyvalue` for **every resolved color** (aliases, -legacy names, derived shades, light/dark mode detection). It is the same resolver the templates use. - -### Recommendation for OmaMovie -- **Read the palette with `omarchy-theme-color --all`**: no TOML parser dependency and the same - resolution as Omarchy. Fallback: read `colors.toml` directly; final fallback: a built-in - palette (the app must work outside Omarchy and in tests). -- **Live switching**: watch `~/.local/state/omarchy/current/` (the `theme` directory is - **replaced** with `mv`, so watch the parent; `theme.name` changes on every switch) with - `QFileSystemWatcher`, debounced. This avoids installing hooks in the user's directory. -- Map the palette to semantic UI tokens (panel background, selection, accent, secondary text) in - a single place (a `ThemeProvider` exposed to QML). -- **Never theme color-critical areas**: preview, scopes, color picker and thumbnails show the - real image. The theme only affects the surrounding interface. -- Fonts: `omarchy-font-current` and the `font-set.d` hook. - ---- - -## 4. Window opacity (critical for color) - -Omarchy applies translucency to **every** window: -```lua -o.window(".*", { tag = "+default-opacity" }) -o.window({ tag = "default-opacity" }, { opacity = "0.985 0.96" }) -``` -Omarchy itself makes an exception for DaVinci Resolve (`default/hypr/apps/davinci-resolve.lua`): -> "Kept fully opaque: the default translucency distorts colour-critical grading work." - -**OmaMovie needs the same exception.** A 96% opaque window blends the wallpaper into the preview -and invalidates any color judgment. - -- The app sets a **stable Wayland app_id**: `omamovie` (through `QGuiApplication::setDesktopFileName("omamovie")`). -- Required rule: - ```lua - o.window("omamovie", { tag = "-default-opacity", opacity = "1 1" }) - ``` -- Where to put it: short term, document it for users (Hyprland config in `~/.config/hypr`); - medium term, **propose upstream** a `default/hypr/apps/omamovie.lua`, as exists for Resolve. -- A Wayland client does not control the opacity applied by the compositor. - ---- - -## 5. Screen recordings - -`omarchy-capture-screenrecording` (default shortcut **ALT+PRINT**) uses `gpu-screen-recorder`: -``` -gpu-screen-recorder ... -k auto -f 60 -fm cfr -fallback-cpu-encoding yes -o -a