From eabb60ae36aa62ee64fa271d0d6ab2e8d4b025ad Mon Sep 17 00:00:00 2001 From: Kxsumi Date: Fri, 11 Sep 2026 00:39:16 -0400 Subject: [PATCH 1/4] feat(video): music-video source with verified matching Switching a track to Video picked the wrong upload often enough to be unusable: YT Music's search results misreport both title and artist, so a lyrics reupload or a fan edit would win on text alone. Candidates are now checked against what YouTube actually says. Every candidate is screened through oembed for its real title and channel, which is cheap enough to run on all of them, and only a shortlist is verified with yt-dlp for the things oembed cannot answer: the verified badge, channel id, category and duration. A candidate is trusted when it sits on the artist's own channel or a verified one, and lyrics or translated reuploads are rejected unless the channel itself checks out. Also: - Song/Video toggle carries across tracks, with the next track's video warmed while the current one plays. - Quality follows what the GPU can decode: auto probes for 4K VP9/AV1 and steps down on dropped frames, or can be pinned from Settings. - The backdrop is aligned to the audio element's own clock and trimmed by playback rate rather than seeking, which is what kept it up to half a second out of step. - Optional SponsorBlock skipping for non-music sections, marked on the progress bar. Off by default, and it only ever sends a hash prefix of the video id. - Full-screen controls fade once the mouse settles. --- src-tauri/capabilities/default.json | 3 +- src-tauri/src/lib.rs | 428 ++++++++++++++++++- src/components/layout/fullscreen-player.tsx | 191 ++++++++- src/components/layout/player-bar.tsx | 41 +- src/components/layout/player-more-menu.tsx | 15 +- src/components/settings/appearance-tab.tsx | 21 + src/components/settings/playback-tab.tsx | 88 ++++ src/components/settings/storage-tab.tsx | 40 ++ src/components/ui/slider.tsx | 29 +- src/lib/audio-clock.ts | 19 + src/lib/audio-engine.ts | 131 +++++- src/lib/diagnostics.ts | 11 + src/lib/innertube/alternate-source.ts | 444 +++++++++++++++++++- src/lib/innertube/counterpart.ts | 42 ++ src/lib/innertube/video-meta.ts | 41 ++ src/lib/source-follow.ts | 63 +++ src/lib/sponsorblock.ts | 95 +++++ src/lib/store/playback-settings.ts | 28 ++ src/lib/store/settings.ts | 5 + src/lib/store/sponsor-segments.ts | 15 + src/lib/store/track-source.ts | 14 +- src/lib/store/video-gate.ts | 19 + src/lib/stream.ts | 38 ++ src/lib/track-meta.ts | 19 + src/lib/video-quality.ts | 59 +++ 25 files changed, 1865 insertions(+), 34 deletions(-) create mode 100644 src/lib/audio-clock.ts create mode 100644 src/lib/diagnostics.ts create mode 100644 src/lib/innertube/counterpart.ts create mode 100644 src/lib/innertube/video-meta.ts create mode 100644 src/lib/source-follow.ts create mode 100644 src/lib/sponsorblock.ts create mode 100644 src/lib/store/sponsor-segments.ts create mode 100644 src/lib/store/video-gate.ts create mode 100644 src/lib/video-quality.ts diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 09605245..30a424cf 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -19,7 +19,8 @@ { "url": "https://itunes.apple.com/*" }, { "url": "https://apic-desktop.musixmatch.com/*" }, { "url": "https://genius.com/*" }, - { "url": "https://lrclib.net/*" } + { "url": "https://lrclib.net/*" }, + { "url": "https://sponsor.ajay.app/*" } ] }, "store:default", diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 98667253..fc2cc35f 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -11,7 +11,7 @@ use tokio::process::Command as TokioCommand; use tokio::sync::{Mutex, Notify}; use axum::{ - extract::{Path, Request, State as AxumState}, + extract::{Path, Query, Request, State as AxumState}, http::StatusCode, response::{IntoResponse, Response}, routing::get, @@ -3228,6 +3228,9 @@ struct StreamServer { /// in-flight independently for the two modes. ephemeral_dir: PathBuf, cover_dir: PathBuf, + /// Music-video files for the full-screen backdrop. Session-only, + /// wiped on startup like the ephemeral audio cache. + video_dir: PathBuf, downloads: DownloadMap, /// Expected location of the managed yt-dlp copy. Resolution to an /// actual program (managed vs PATH fallback) happens per-spawn via @@ -3954,12 +3957,415 @@ fn generate_stream_token() -> String { out } +const VIDEO_FORMAT_LO: &str = "bestvideo[vcodec^=avc1][height<=1440][protocol^=http]/bestvideo[height<=1440][protocol^=http]/bestvideo[protocol^=http]/best[protocol^=http]"; +const VIDEO_FORMAT_HI: &str = "bestvideo[vcodec^=vp9][height<=2160][protocol^=http]/bestvideo[vcodec^=av01][height<=2160][protocol^=http]/bestvideo[vcodec^=avc1][height<=1440][protocol^=http]/bestvideo[height<=1440][protocol^=http]/bestvideo[protocol^=http]/best[protocol^=http]"; + +fn spawn_video_downloader( + video_id: String, + tier: String, + target_dir: PathBuf, + map_key: String, + srv: StreamServer, + state: Arc, +) { + let downloads = srv.downloads.clone(); + tokio::spawn(async move { + let url = format!("https://www.youtube.com/watch?v={video_id}"); + let part_path = target_dir.join(format!("{video_id}.{tier}.video.part")); + let final_path = target_dir.join(format!("{video_id}.{tier}.video.webm")); + let _ = tokio::fs::create_dir_all(&target_dir).await; + let _ = tokio::fs::remove_file(&part_path).await; + + let format = if tier == "hi" { VIDEO_FORMAT_HI } else { VIDEO_FORMAT_LO }; + let mut cmd = TokioCommand::new(ytdlp::program(&srv.ytdlp_bin)); + cmd.args([ + "-f", + format, + "--no-playlist", + "--no-warnings", + "--no-part", + "-q", + "--retries", + "5", + "--extractor-retries", + "3", + "--socket-timeout", + "15", + "-o", + "-", + ]); + cmd.arg(&url); + #[cfg(windows)] + cmd.creation_flags(0x0800_0000); + let mut child = match cmd.stdout(Stdio::piped()).stderr(Stdio::inherit()).spawn() { + Ok(c) => c, + Err(e) => { + eprintln!("[stream-video] spawn {video_id}: {e}"); + state.complete.store(true, Ordering::Release); + state.notify.notify_waiters(); + downloads.lock().await.remove(&map_key); + return; + } + }; + + let mut stdout = child.stdout.take().unwrap(); + let mut file = tokio::fs::File::create(&part_path).await.ok(); + let mut buf = vec![0u8; 64 * 1024]; + let mut ok = true; + const READ_TIMEOUT: Duration = Duration::from_secs(60); + loop { + match tokio::time::timeout(READ_TIMEOUT, stdout.read(&mut buf)).await { + Err(_) => { + eprintln!("[stream-video] read timeout for {video_id}; killing yt-dlp"); + let _ = child.start_kill(); + ok = false; + break; + } + Ok(Ok(0)) => break, + Ok(Ok(n)) => { + let chunk = &buf[..n]; + if let Some(ref mut f) = file { + if let Err(e) = f.write_all(chunk).await { + eprintln!("[stream-video] write .part: {e}"); + file = None; + ok = false; + } + } + state.notify.notify_waiters(); + } + Ok(Err(e)) => { + eprintln!("[stream-video] read stdout: {e}"); + ok = false; + break; + } + } + } + if let Some(mut f) = file.take() { + let _ = f.flush().await; + drop(f); + } + let status = child.wait().await; + let success = ok && status.map(|s| s.success()).unwrap_or(false); + + const MIN_VIDEO_BYTES: u64 = 32 * 1024; + let part_size = tokio::fs::metadata(&part_path) + .await + .map(|m| m.len()) + .unwrap_or(0); + if success && part_size >= MIN_VIDEO_BYTES { + if let Err(e) = tokio::fs::rename(&part_path, &final_path).await { + eprintln!("[stream-video] rename: {e}"); + let _ = tokio::fs::remove_file(&part_path).await; + } else { + eprintln!("[stream-video] cached {video_id} ({part_size} bytes)"); + } + } else { + if success { + eprintln!( + "[stream-video] download too small for {video_id}: {part_size} bytes (min {MIN_VIDEO_BYTES})" + ); + } else { + eprintln!("[stream-video] download failed {video_id}"); + } + let _ = tokio::fs::remove_file(&part_path).await; + } + + state.complete.store(true, Ordering::Release); + state.notify.notify_waiters(); + + if success { + let downloads_evict = downloads.clone(); + let key = map_key.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(60)).await; + downloads_evict.lock().await.remove(&key); + }); + } else { + downloads.lock().await.remove(&map_key); + } + }); +} + +async fn sniff_video_mime(path: &std::path::Path) -> &'static str { + let mut buf = [0u8; 16]; + if let Ok(mut f) = tokio::fs::File::open(path).await { + let _ = f.read(&mut buf).await; + } + if &buf[4..8] == b"ftyp" { + "video/mp4" + } else { + "video/webm" + } +} + +async fn video_handler( + AxumState(srv): AxumState, + Path(video_id): Path, + Query(params): Query>, + req: Request, +) -> Response { + if !sanitize_video_id(&video_id) { + return (StatusCode::BAD_REQUEST, "invalid videoId").into_response(); + } + + let tier = match params.get("q").map(|s| s.as_str()) { + Some("hi") => "hi", + _ => "lo", + }; + let target_dir = srv.video_dir.clone(); + let map_key = format!("v:{tier}:{video_id}"); + let final_path = target_dir.join(format!("{video_id}.{tier}.video.webm")); + + let t0 = std::time::Instant::now(); + let range_hdr = req + .headers() + .get(axum::http::header::RANGE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + eprintln!( + "[stream-video] GET /video/{video_id} range={range_hdr:?} cached={}", + final_path.exists() + ); + + if !final_path.exists() { + let state = { + let mut map = srv.downloads.lock().await; + if let Some(s) = map.get(&map_key) { + s.clone() + } else { + let s = Arc::new(DownloadState { + complete: Arc::new(AtomicBool::new(false)), + notify: Arc::new(Notify::new()), + }); + map.insert(map_key.clone(), s.clone()); + drop(map); + spawn_video_downloader( + video_id.clone(), + tier.to_string(), + target_dir.clone(), + map_key.clone(), + srv.clone(), + s.clone(), + ); + s + } + }; + + let deadline = tokio::time::Instant::now() + Duration::from_secs(180); + while !state.complete.load(Ordering::Acquire) { + if tokio::time::Instant::now() >= deadline { + eprintln!("[stream-video] {video_id}: TIMEOUT after 180s"); + return (StatusCode::GATEWAY_TIMEOUT, "download timeout").into_response(); + } + let notified = state.notify.notified(); + tokio::pin!(notified); + let _ = tokio::time::timeout(Duration::from_secs(5), notified).await; + } + + if !final_path.exists() { + eprintln!( + "[stream-video] {video_id}: BAD_GATEWAY, complete but no file (elapsed {:.2}s)", + t0.elapsed().as_secs_f32() + ); + return (StatusCode::BAD_GATEWAY, "download failed").into_response(); + } + eprintln!( + "[stream-video] {video_id}: download finished in {:.2}s", + t0.elapsed().as_secs_f32() + ); + } + + let sniffed_ct = sniff_video_mime(&final_path).await; + let mut resp = ServeFile::new(&final_path) + .oneshot(req) + .await + .map(|r| r.into_response()) + .unwrap_or_else(|e| { + (StatusCode::INTERNAL_SERVER_ERROR, format!("serve: {e}")).into_response() + }); + if resp.status().is_success() || resp.status() == StatusCode::PARTIAL_CONTENT { + resp.headers_mut().insert( + axum::http::header::CONTENT_TYPE, + axum::http::HeaderValue::from_static(sniffed_ct), + ); + } + eprintln!( + "[stream-video] {video_id}: responding {} ({:.2}s total) ct={:?} len={:?}", + resp.status(), + t0.elapsed().as_secs_f32(), + resp.headers() + .get(axum::http::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()), + resp.headers() + .get(axum::http::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()), + ); + resp +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct VideoMeta { + title: Option, + channel: Option, + channel_id: Option, + channel_is_verified: Option, + categories: Vec, + view_count: Option, + duration: Option, +} + +/// Title and uploader for one video, straight from YouTube's oembed +/// endpoint. Two orders of magnitude cheaper than a yt-dlp extraction +/// (one small HTTP GET, no process spawn), so every candidate can afford +/// one; the fields yt-dlp alone can answer (verified badge, channel id, +/// category) still need `probe_video_meta` on the finalists. +#[derive(serde::Serialize)] +struct VideoBrief { + id: String, + title: Option, + channel: Option, +} + +#[tauri::command] +async fn probe_video_briefs(video_ids: Vec) -> Result, String> { + const MAX_BRIEFS: usize = 40; + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(8)) + .build() + .map_err(|e| format!("build brief client: {e}"))?; + let mut handles = Vec::new(); + for id in video_ids + .into_iter() + .filter(|v| sanitize_video_id(v)) + .take(MAX_BRIEFS) + { + let client = client.clone(); + handles.push(tokio::spawn(async move { + let url = format!( + "https://www.youtube.com/oembed?url=https%3A//www.youtube.com/watch%3Fv%3D{id}&format=json" + ); + let mut brief = VideoBrief { + id, + title: None, + channel: None, + }; + let Ok(res) = client.get(&url).send().await else { + return brief; + }; + if !res.status().is_success() { + return brief; + } + let Ok(body) = res.text().await else { + return brief; + }; + let Ok(json) = serde_json::from_str::(&body) else { + return brief; + }; + brief.title = json.get("title").and_then(|v| v.as_str()).map(String::from); + brief.channel = json + .get("author_name") + .and_then(|v| v.as_str()) + .map(String::from); + brief + })); + } + let mut out = Vec::new(); + for h in handles { + if let Ok(brief) = h.await { + out.push(brief); + } + } + Ok(out) +} + +#[tauri::command] +async fn probe_video_meta(app: tauri::AppHandle, video_id: String) -> Result { + if !sanitize_video_id(&video_id) { + return Err(format!("invalid videoId: {video_id}")); + } + let url = format!("https://www.youtube.com/watch?v={video_id}"); + let mut command = TokioCommand::new(ytdlp::program(&ytdlp::managed_path(&app))); + command.args([ + "-j", + "--skip-download", + "--no-playlist", + "--no-warnings", + "--socket-timeout", + "10", + &url, + ]); + #[cfg(windows)] + command.creation_flags(0x0800_0000); // CREATE_NO_WINDOW + let output = command + .output() + .await + .map_err(|e| format!("spawn yt-dlp: {e}"))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "yt-dlp exit {}: {}", + output.status, + stderr.chars().take(400).collect::() + )); + } + let json: serde_json::Value = + serde_json::from_slice(&output.stdout).map_err(|e| format!("parse: {e}"))?; + Ok(VideoMeta { + title: json.get("title").and_then(|v| v.as_str()).map(String::from), + channel: json + .get("channel") + .and_then(|v| v.as_str()) + .map(String::from), + channel_id: json + .get("channel_id") + .and_then(|v| v.as_str()) + .map(String::from), + channel_is_verified: json.get("channel_is_verified").and_then(|v| v.as_bool()), + categories: json + .get("categories") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|x| x.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(), + view_count: json.get("view_count").and_then(|v| v.as_u64()), + duration: json.get("duration").and_then(|v| v.as_f64()), + }) +} + +async fn wipe_dir_files(dir: &std::path::Path, label: &str) { + if let Ok(mut rd) = tokio::fs::read_dir(dir).await { + let mut wiped: u64 = 0; + while let Ok(Some(entry)) = rd.next_entry().await { + if let Ok(meta) = entry.metadata().await { + if meta.is_file() { + wiped += meta.len(); + let _ = tokio::fs::remove_file(entry.path()).await; + } + } + } + if wiped > 0 { + eprintln!("[stream-server] wiped {wiped} bytes from {label} dir"); + } + } +} + +#[tauri::command] +async fn diag_log(tag: String, text: String) { + eprintln!("[{tag}] {text}"); +} + async fn start_stream_server( port_state: Arc>>, token_state: Arc>>, cache_dir: PathBuf, ephemeral_dir: PathBuf, cover_dir: PathBuf, + video_dir: PathBuf, ytdlp_bin: PathBuf, ) { if let Err(e) = tokio::fs::create_dir_all(&cache_dir).await { @@ -3971,6 +4377,10 @@ async fn start_stream_server( if let Err(e) = tokio::fs::create_dir_all(&cover_dir).await { eprintln!("[stream-server] mkdir {cover_dir:?}: {e}"); } + if let Err(e) = tokio::fs::create_dir_all(&video_dir).await { + eprintln!("[stream-server] mkdir {video_dir:?}: {e}"); + } + wipe_dir_files(&video_dir, "video").await; // Wipe whatever a previous (anonymous / Free) session left behind. // Persisting tracks across restarts is a Premium-only feature; if a @@ -3995,6 +4405,7 @@ async fn start_stream_server( cache_dir, ephemeral_dir, cover_dir, + video_dir, downloads: Arc::new(Mutex::new(HashMap::new())), ytdlp_bin, }; @@ -4008,6 +4419,7 @@ async fn start_stream_server( let routes = Router::new() .route("/stream/:video_id", get(stream_handler)) + .route("/video/:video_id", get(video_handler)) .route("/prefetch/:video_id", get(prefetch_handler)) .route("/cover/:filename", get(cover_serve_handler)) .with_state(server); @@ -4185,6 +4597,9 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ ensure_ytdlp, resolve_stream_ytdlp, + probe_video_meta, + probe_video_briefs, + diag_log, get_stream_base_url, start_login, get_cookie_header, @@ -4296,6 +4711,7 @@ pub fn run() { let cache_dir = cache_root.join("stream"); let ephemeral_dir = cache_root.join("stream-ephemeral"); let cover_dir = cache_root.join("covers"); + let video_dir = cache_root.join("stream-video"); let handle = app.handle().clone(); eprintln!("[stream-server] cache dir: {cache_dir:?}"); eprintln!("[stream-server] ephemeral dir: {ephemeral_dir:?}"); @@ -4308,7 +4724,15 @@ pub fn run() { // email-based dedup before the UI reads the list. dedup_accounts_by_identity(&handle).await; cleanup_login_artifacts(&handle).await; - start_stream_server(port, token, cache_dir, ephemeral_dir, cover_dir, ytdlp_bin) + start_stream_server( + port, + token, + cache_dir, + ephemeral_dir, + cover_dir, + video_dir, + ytdlp_bin, + ) .await; }); // Subscribe to resume-from-sleep before the loop starts, so a diff --git a/src/components/layout/fullscreen-player.tsx b/src/components/layout/fullscreen-player.tsx index 643ff226..f73343ca 100644 --- a/src/components/layout/fullscreen-player.tsx +++ b/src/components/layout/fullscreen-player.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { HeartIcon, ThumbsDownIcon, ThumbsUpIcon } from "lucide-react"; import { toast } from "sonner"; @@ -48,6 +48,19 @@ import { useFullscreenStore } from "@/lib/store/fullscreen"; import { usePlaybackStore, currentTrack } from "@/lib/store/playback"; import { useScrubStore } from "@/lib/store/scrub"; import { useSettingsStore, type FullscreenLayout } from "@/lib/store/settings"; +import { useTrackSourceStore } from "@/lib/store/track-source"; +import { videoStreamUrlFor } from "@/lib/stream"; +import { getVideoQualityTier, reportPlaybackStutter } from "@/lib/video-quality"; +import { useVideoGateStore } from "@/lib/store/video-gate"; +import { getAudioClock } from "@/lib/audio-clock"; + +const VIDEO_WAIT_CEILING_MS = 45_000; +/** Under this the picture reads as in step, so leave the rate alone. */ +const SYNC_TOLERANCE = 0.04; +/** Past this it's a seek or a stall, not drift, so jump rather than ease. */ +const SYNC_SEEK_THRESHOLD = 0.5; +const SYNC_MAX_RATE_TRIM = 0.1; +const IDLE_DELAY_MS = 2800; import type { QueueTrack } from "@/lib/store/playback"; /** @@ -102,6 +115,99 @@ const sideButton = (on: boolean) => /* The view */ /* ------------------------------------------------------------------ */ +function VideoBackdrop({ + streamId, + onError, + onReady, +}: { + streamId: string; + onError: () => void; + onReady: () => void; +}) { + const [src, setSrc] = useState(null); + const videoRef = useRef(null); + + useEffect(() => { + let cancelled = false; + setSrc(null); + (async () => { + const tier = await getVideoQualityTier(); + const url = await videoStreamUrlFor(streamId, tier); + if (!cancelled) setSrc(url); + })(); + return () => { + cancelled = true; + }; + }, [streamId]); + + useEffect(() => { + if (!src) return; + const check = setTimeout(() => { + const el = videoRef.current; + if (!el || typeof el.getVideoPlaybackQuality !== "function") return; + const q = el.getVideoPlaybackQuality(); + if (q.totalVideoFrames > 60 && q.droppedVideoFrames / q.totalVideoFrames > 0.1) { + reportPlaybackStutter(); + } + }, 4000); + return () => clearTimeout(check); + }, [src]); + + useEffect(() => { + if (!src) return; + let raf: number; + const tick = () => { + const el = videoRef.current; + if (el) { + const { position, playing } = usePlaybackStore.getState(); + if (playing && el.paused) void el.play().catch(() => {}); + if (!playing && !el.paused) el.pause(); + const target = getAudioClock() ?? position; + const seekable = !el.duration || target <= el.duration; + if (el.readyState >= 2 && seekable) { + const drift = el.currentTime - target; + const off = Math.abs(drift); + if (off > SYNC_SEEK_THRESHOLD || !playing) { + if (off > SYNC_TOLERANCE) el.currentTime = target; + if (el.playbackRate !== 1) el.playbackRate = 1; + } else if (off > SYNC_TOLERANCE) { + // Ease the picture back onto the audio instead of jumping. + // A seek this small costs a visible hitch and re-drifts within + // seconds; the rate change is imperceptible on a muted video. + const trim = Math.max( + -SYNC_MAX_RATE_TRIM, + Math.min(SYNC_MAX_RATE_TRIM, drift), + ); + el.playbackRate = 1 - trim; + } else if (el.playbackRate !== 1) { + el.playbackRate = 1; + } + } + } + raf = requestAnimationFrame(tick); + }; + raf = requestAnimationFrame(tick); + return () => cancelAnimationFrame(raf); + }, [src]); + + if (!src) return null; + + return ( +