From 2ce081946477df50dc73a8ada091c609758513e1 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Wed, 21 Oct 2026 04:56:48 +0200 Subject: [PATCH 1/4] build(deps): bump waveflow-core to 102df003 The pin moves from WaveFlow #883 (9b50f3de, 2026-10-17) to the main of 2026-10-21 (102df003, #926), 167 commits later. What the server takes from the core is unchanged by them apart from `codec_label` moving under `audio_format`, which nothing here names. The one addition worth having is `text_encoding` (WaveFlow #910): a decoder for the text files users bring, which the next two commits put behind sidecar lyrics and playlist import. `default-features = false` stays, and the new `playback` feature stays off: the server plays nothing, and it would pull in `rubato`. lofty (0.25.4) and symphonia (0.6.1) are unchanged, and `cargo tree -i` shows one copy of each. The only crates the bump adds are `chardetng`, `encoding_rs` and their small helpers. Signed-off-by: InstaZDLL --- Cargo.lock | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 +- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b2496e2..958f6931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -432,6 +432,17 @@ dependencies = [ "poly1305", ] +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.45" @@ -551,6 +562,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -749,6 +766,20 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1645,6 +1676,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2444,6 +2481,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simplecss" version = "0.2.2" @@ -2932,7 +2975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3558,12 +3601,14 @@ dependencies = [ [[package]] name = "waveflow-core" version = "1.9.1" -source = "git+https://github.com/InstaZDLL/WaveFlow?rev=9b50f3de288d62b0ff9f867ab7bda504b255cc5c#9b50f3de288d62b0ff9f867ab7bda504b255cc5c" +source = "git+https://github.com/InstaZDLL/WaveFlow?rev=102df0031bef8ba44062e03337807592b3d51102#102df0031bef8ba44062e03337807592b3d51102" dependencies = [ "anyhow", "async-trait", "blake3", + "chardetng", "chrono", + "encoding_rs", "fast_image_resize", "globset", "hex", @@ -3585,6 +3630,7 @@ dependencies = [ "tracing", "usvg", "uuid", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 845e927c..a10fe185 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ utoipa-scalar = { version = "0.4", features = ["axum"] } url = "2" uuid = { version = "1", features = ["v4", "v8", "serde"] } walkdir = "2" -waveflow-core = { git = "https://github.com/InstaZDLL/WaveFlow", rev = "9b50f3de288d62b0ff9f867ab7bda504b255cc5c", default-features = false } +waveflow-core = { git = "https://github.com/InstaZDLL/WaveFlow", rev = "102df0031bef8ba44062e03337807592b3d51102", default-features = false } [dev-dependencies] http-body-util = "0.1" From 29b47e9fe2a2d14eb3901c77ce3c8f22f4094762 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Wed, 21 Oct 2026 05:00:37 +0200 Subject: [PATCH 2/4] fix(scanner): read sidecar lyrics that are not utf-8 `read_sidecar` ended in `String::from_utf8(bytes).ok()`, so an `.lrc` or `.txt` in a system code page (GB18030 on a Chinese Windows, Big5, Shift_JIS, EUC-KR, a Western Windows code page) was dropped whole and the track reached every client without lyrics. A UTF-8 file with a byte-order mark was read, but kept the mark in front of its first timestamp. It now decodes with `waveflow_core::text_encoding::decode`, the decoder the desktop reads the same files with since WaveFlow #910: a byte-order mark names the encoding and is dropped, valid UTF-8 is returned as it is, and anything else has its encoding guessed by chardetng. The size cap and the symlink refusal are untouched. Tests: a Shift_JIS sidecar and a marked UTF-8 one through `read_sidecar`, and a GB18030 `.lrc` through a scan to the lyrics service. Both fail on the old decoding, the first on the missing lyrics and the second on an empty `structured_lyrics`. The `[0xff]` case is gone from the rejection test: such a file is now read, which is the change. Signed-off-by: InstaZDLL --- Cargo.lock | 1 + Cargo.toml | 1 + docs/api-v2-guide.md | 4 ++- docs/subsonic-api-guide.md | 6 ++-- src/scanner.rs | 36 ++++++++++++++++--- tests/scanner.rs | 71 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 958f6931..47eef8c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3649,6 +3649,7 @@ dependencies = [ "clap", "dashmap", "dotenvy", + "encoding_rs", "futures-util", "hex", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index a10fe185..c5f03eb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ walkdir = "2" waveflow-core = { git = "https://github.com/InstaZDLL/WaveFlow", rev = "102df0031bef8ba44062e03337807592b3d51102", default-features = false } [dev-dependencies] +encoding_rs = "0.8.42" http-body-util = "0.1" tempfile = "3" tokio-tungstenite = "0.30" diff --git a/docs/api-v2-guide.md b/docs/api-v2-guide.md index 17d0333f..e0b5e6a4 100644 --- a/docs/api-v2-guide.md +++ b/docs/api-v2-guide.md @@ -424,7 +424,9 @@ its hash — a shared cache keyed on the URL would hand one tenant's artwork to another with no credential presented. A private cache is the client's own and loses nothing by the restriction. -Lyrics return embedded or UTF-8 `.lrc`/`.txt` sidecar content. Synchronized +Lyrics return embedded or `.lrc`/`.txt` sidecar content. A sidecar need not be +UTF-8: a byte-order mark names its encoding, and otherwise one such as GB18030, +Big5, Shift_JIS or a Windows code page is recognised from the bytes. Synchronized line starts are milliseconds; plain lines omit `start`: ```bash diff --git a/docs/subsonic-api-guide.md b/docs/subsonic-api-guide.md index fd839843..75896d9b 100644 --- a/docs/subsonic-api-guide.md +++ b/docs/subsonic-api-guide.md @@ -193,8 +193,10 @@ curl -X POST https://music.example.com/rest/getLyricsBySongId.view \ --data-urlencode "id=TRACK_UUID" ``` -WaveFlow returns embedded text and UTF-8 `.lrc`/`.txt` sidecars. LRC timestamps -are milliseconds. The legacy `getLyrics` exact artist/title lookup is also +WaveFlow returns embedded text and `.lrc`/`.txt` sidecars. A sidecar need not +be UTF-8: a byte-order mark names its encoding, and otherwise one such as +GB18030, Big5, Shift_JIS or a Windows code page is recognised from the bytes. +LRC timestamps are milliseconds. The legacy `getLyrics` exact artist/title lookup is also implemented. Stream an original with a byte range: diff --git a/src/scanner.rs b/src/scanner.rs index 32921f7b..ac276593 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -970,7 +970,11 @@ fn read_sidecar(path: &Path) -> Option { if bytes.len() as u64 > MAX_LYRICS_BYTES { return None; } - String::from_utf8(bytes).ok() + // Not `String::from_utf8`: lyrics written by a tool on a Chinese, Japanese + // or Korean Windows are in that system's code page, and a strict UTF-8 + // read dropped them whole. The core decoder is the one the desktop reads + // the same files with, so both answer the same text for them. + Some(waveflow_core::text_encoding::decode(&bytes).into_owned()) } fn lyrics_hash(entries: &[LyricsInput]) -> String { @@ -1082,9 +1086,6 @@ mod tests { std::fs::write(&oversized, vec![b'x'; MAX_LYRICS_BYTES as usize + 1]).unwrap(); assert!(read_sidecar(&oversized).is_none()); - let invalid_utf8 = temp.path().join("invalid.lrc"); - std::fs::write(&invalid_utf8, [0xff]).unwrap(); - assert!(read_sidecar(&invalid_utf8).is_none()); assert!(read_sidecar(temp.path()).is_none()); let link = temp.path().join("linked.lrc"); @@ -1096,4 +1097,31 @@ mod tests { assert!(read_sidecar(&link).is_none()); } } + + /// Lyrics saved by a tool on a Japanese Windows, in that system's code + /// page. A strict UTF-8 read dropped the file whole, so the track had no + /// lyrics at all. + #[test] + fn sidecar_lyrics_that_are_not_utf8_are_read_in_their_own_encoding() { + let temp = tempfile::tempdir().unwrap(); + let text = "[00:01.00]今日はいい天気ですね、一緒に公園を散歩しましょう\n\ + [00:05.50]湖に光が映って、風が森を通り抜ける音がする"; + let (bytes, _, unmappable) = encoding_rs::SHIFT_JIS.encode(text); + assert!(!unmappable, "Shift_JIS holds the sample"); + assert!( + std::str::from_utf8(&bytes).is_err(), + "the sample is not UTF-8 by accident" + ); + let japanese = temp.path().join("japanese.lrc"); + std::fs::write(&japanese, &bytes).unwrap(); + assert_eq!(read_sidecar(&japanese).as_deref(), Some(text)); + + // A byte-order mark names the encoding and is not part of the text. + // Kept, it sat in front of the first timestamp. + let marked = temp.path().join("marked.lrc"); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(b"[00:01]marked"); + std::fs::write(&marked, bytes).unwrap(); + assert_eq!(read_sidecar(&marked).as_deref(), Some("[00:01]marked")); + } } diff --git a/tests/scanner.rs b/tests/scanner.rs index ef814efa..9e7d153b 100644 --- a/tests/scanner.rs +++ b/tests/scanner.rs @@ -109,6 +109,77 @@ async fn scanner_indexes_moves_and_marks_tracks_unavailable() { assert!(!tracks[0].available); } +/// Sidecar lyrics in GB18030, the way a tool on a Chinese Windows saves them +/// (WaveFlow#910). The scanner read sidecars as strict UTF-8 and dropped this +/// one whole, so the track reached every client with no lyrics. +#[tokio::test] +async fn sidecar_lyrics_that_are_not_utf8_reach_the_lyrics_service() { + let (_temp, config, state) = test_app().await; + // Inserted rather than created: a scan needs an owner, not a login, and + // hashing a literal password is what `rust/hard-coded-cryptographic-value` + // reports. + let owner = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO account (id, username, password_hash, role, disabled, created_at, updated_at) \ + VALUES (?, ?, 'this-account-never-authenticates', 'admin', 0, ?, ?)", + ) + .bind(owner.to_string()) + .bind("scanner-encodings") + .bind(now_ms()) + .bind(now_ms()) + .execute(state.db.pool()) + .await + .unwrap(); + let music = config.data_dir.join("encoded-lyrics"); + std::fs::create_dir_all(&music).unwrap(); + write_test_wav(&music.join("Song.wav")); + let (bytes, _, unmappable) = encoding_rs::GB18030 + .encode("[00:01.00]今天的天气很好，我们一起去公园散步吧\n[00:05.50]阳光照在湖面上"); + assert!(!unmappable); + assert!( + std::str::from_utf8(&bytes).is_err(), + "the sample is not UTF-8 by accident" + ); + std::fs::write(music.join("Song.lrc"), &bytes).unwrap(); + let root = std::fs::canonicalize(&music).unwrap(); + let library_id = state + .db + .create_library( + owner, + "Encoded lyrics", + &root, + LibraryVisibility::Private, + now_ms(), + ) + .await + .unwrap(); + let library = LibraryRecord { + id: library_id, + name: "Encoded lyrics".into(), + root_path: root, + }; + + run_scan(&state, owner, library).await; + let tracks = state + .db + .list_tracks_for_user(owner, library_id) + .await + .unwrap(); + assert_eq!(tracks.len(), 1); + let lyrics = state.services.lyrics(owner, tracks[0].id).await.unwrap(); + assert_eq!( + lyrics.structured_lyrics.len(), + 1, + "the sidecar was read rather than dropped" + ); + let lines = &lyrics.structured_lyrics[0].lines; + assert!(lyrics.structured_lyrics[0].synced); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0].start, Some(1_000)); + assert_eq!(lines[0].value, "今天的天气很好，我们一起去公园散步吧"); + assert_eq!(lines[1].value, "阳光照在湖面上"); +} + #[tokio::test] async fn scanner_batches_more_than_one_write_group_without_deduplicating_copies() { let (_temp, config, state) = test_app().await; From c27e66aec1d9da335f1064e2d6b505748eb46234 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Wed, 21 Oct 2026 05:00:59 +0200 Subject: [PATCH 3/4] fix(playlists): read imported playlists with the core decoder `playlist_file::decode` read UTF-8 when the bytes were UTF-8 and Windows-1252 otherwise. A playlist written on a Chinese, Japanese or Korean Windows came back as Latin nonsense that matched no path, and a UTF-16 one, as Notepad saves "Unicode", came back with a NUL between every character. Both were reported as read, and both resolved nothing. The text is now `waveflow_core::text_encoding::decode`, the decoder the desktop uses: a byte-order mark names the encoding, valid UTF-8 is UTF-8, and anything else has its encoding guessed by chardetng. The report still says which encoding was used, which the core does not answer, so `encoding_read_as` works it out by the same steps with the detector set up the same way. A test decodes samples in six encodings with the encoding it names and compares against the core, so a core that changes its guess fails there instead of having a report name one encoding while the text was read as another. The wire contract is extended on purpose. `encoding` was `utf-8` or `windows-1252`. It is now the encoding's WHATWG name in lower case, an open set (`utf-16le`, `shift_jis`, `gbk`, `big5`, `euc-kr`, ...), and the two old values are spelled as they were. The schema says `string`. The web client already showed any value other than `utf-8`, so only its type changes. Tests: a UTF-16 LE file with its mark and a Shift_JIS file through `parse`, and a UTF-16 LE import through `/api/v2/playlists/import`. All three fail on the old decoding. Signed-off-by: InstaZDLL --- Cargo.lock | 1 + Cargo.toml | 7 +- docs/api-v2-guide.md | 8 +- src/playlist_file.rs | 197 ++++++++++++++++------ src/services/mod.rs | 7 +- tests/native_api.rs | 23 +++ webapp/src/api/playlists.ts | 8 +- webapp/src/components/playlist-import.tsx | 8 +- 8 files changed, 200 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47eef8c6..90f74f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3645,6 +3645,7 @@ dependencies = [ "blake3", "bytes", "chacha20poly1305", + "chardetng", "chrono", "clap", "dashmap", diff --git a/Cargo.toml b/Cargo.toml index c5f03eb6..5c2dda97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,10 +26,16 @@ base64 = "0.23" blake3 = "1" bytes = "1" chacha20poly1305 = "0.11" +# Both arrive with `waveflow-core`, whose `text_encoding` decodes the text +# files users bring. Named here so an imported playlist can say which +# encoding it was read as (`playlist_file`), which the core does not answer. +# Same versions as the core's, so nothing new is compiled. +chardetng = "1" chrono = "0.4" clap = { version = "4", features = ["derive"] } dashmap = "6" dotenvy = "0.15" +encoding_rs = "0.8.42" futures-util = "0.3" hex = "0.4" lofty = "0.25" @@ -78,7 +84,6 @@ walkdir = "2" waveflow-core = { git = "https://github.com/InstaZDLL/WaveFlow", rev = "102df0031bef8ba44062e03337807592b3d51102", default-features = false } [dev-dependencies] -encoding_rs = "0.8.42" http-body-util = "0.1" tempfile = "3" tokio-tungstenite = "0.30" diff --git a/docs/api-v2-guide.md b/docs/api-v2-guide.md index e0b5e6a4..7ef65f9a 100644 --- a/docs/api-v2-guide.md +++ b/docs/api-v2-guide.md @@ -740,8 +740,12 @@ name `comment` in `clear`. Shares use the same rule for `description` and JSON. The encoding of a playlist file is part of the problem (`.m3u` is frequently not UTF-8), so the bytes have to arrive as bytes; a JSON envelope would have forced the caller to decode them first, which is the one decision it -cannot make correctly. The server reads UTF-8 when the bytes are UTF-8 and -Windows-1252 otherwise, and **says which it used** in the response. +cannot make correctly. A byte-order mark names the encoding (UTF-8, UTF-16 LE or +BE); otherwise the server reads UTF-8 when the bytes are UTF-8 and recognises +the encoding from the bytes when they are not — GB18030, Big5, Shift_JIS, +EUC-KR, Windows-1252 and the like — and **says which it used** in the response, +as the encoding's WHATWG name in lower case (`utf-8`, `utf-16le`, `shift_jis`, +`windows-1252`…). ```bash # What it would do, without doing it diff --git a/src/playlist_file.rs b/src/playlist_file.rs index 90e69e23..be21fa67 100644 --- a/src/playlist_file.rs +++ b/src/playlist_file.rs @@ -11,24 +11,36 @@ //! `#EXTINF` metadata is kept because it is the only thing left to match on //! when a path resolves to nothing. -use serde::Serialize; -use utoipa::ToSchema; +use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; +use encoding_rs::Encoding; +use serde::{Serialize, Serializer}; /// How the bytes were read as text. /// /// Reported rather than assumed, because the guess can be wrong and a wrong -/// guess mangles every accented title. A caller that sees `Windows1252` on a -/// file it believed was UTF-8 has been told something. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, ToSchema)] -#[serde(rename_all = "kebab-case")] -pub enum TextEncoding { - // Named rather than derived, both of them: kebab-case puts no hyphen before - // a digit, so the derive spelled these "utf8" and "windows1252" and no - // encoding is called either. - #[serde(rename = "utf-8")] - Utf8, - #[serde(rename = "windows-1252")] - Windows1252, +/// guess mangles every title. A caller that sees `shift_jis` on a file it +/// believed was UTF-8 has been told something. +/// +/// On the wire, the encoding's WHATWG name in lower case: `utf-8`, +/// `utf-16le`, `windows-1252`, `shift_jis`, `gbk`, `big5`, `euc-kr` and so on. +/// The two this answered before it read anything but UTF-8 and Windows-1252 +/// are spelled as they were. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TextEncoding(&'static Encoding); + +impl TextEncoding { + pub const UTF_8: Self = Self(encoding_rs::UTF_8); + + /// The name a report gives, `utf-8` rather than encoding_rs's `UTF-8`. + pub fn name(self) -> String { + self.0.name().to_ascii_lowercase() + } +} + +impl Serialize for TextEncoding { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.name()) + } } /// One line of the file that names something to play. @@ -122,44 +134,39 @@ pub fn parse(bytes: &[u8]) -> PlaylistFile { } } -/// UTF-8 if it is UTF-8, Windows-1252 otherwise. +/// The text, read by `waveflow_core::text_encoding`, and the encoding it was +/// read as. /// /// `.m3u8` is UTF-8 by definition and `.m3u` frequently is not, but the /// extension is a claim and the bytes are evidence, so the bytes decide. The -/// order is what makes the fallback safe: valid UTF-8 is never reinterpreted, -/// and accented text in Windows-1252 is not valid UTF-8, so it cannot be -/// mistaken the other way round either. -/// -/// Windows-1252 rather than Latin-1 because the files that are not UTF-8 were -/// written by Windows players: the two agree from `0xA0` up, and below it -/// Latin-1 has control codes where Windows-1252 has the typographic -/// punctuation that actually appears in track titles. +/// core decoder is the one the desktop reads the same files with: a +/// byte-order mark names the encoding, valid UTF-8 is UTF-8, and anything +/// else has its encoding guessed. A fixed fallback, Windows-1252 as this was +/// until the core had a decoder, read a CJK playlist into Latin nonsense and +/// a UTF-16 one into NULs. fn decode(bytes: &[u8]) -> (String, TextEncoding) { - let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes); - match std::str::from_utf8(bytes) { - Ok(text) => (text.to_owned(), TextEncoding::Utf8), - Err(_) => ( - bytes.iter().map(|byte| windows_1252_char(*byte)).collect(), - TextEncoding::Windows1252, - ), - } + let text = waveflow_core::text_encoding::decode(bytes).into_owned(); + (text, TextEncoding(encoding_read_as(bytes))) } -/// The thirty-two code points Windows-1252 puts where Latin-1 keeps C1 -/// control codes. Everything else in the range is Latin-1, which is the -/// identity mapping onto Unicode. -fn windows_1252_char(byte: u8) -> char { - const HIGH: [char; 32] = [ - '\u{20AC}', '\u{0081}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', - '\u{2021}', '\u{02C6}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{008D}', - '\u{017D}', '\u{008F}', '\u{0090}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}', - '\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}', '\u{203A}', - '\u{0153}', '\u{009D}', '\u{017E}', '\u{0178}', - ]; - match byte { - 0x80..=0x9F => HIGH[(byte - 0x80) as usize], - other => char::from(other), +/// The encoding `waveflow_core::text_encoding::decode` reads `bytes` as. +/// +/// The core answers the text and not its encoding, so the encoding is worked +/// out again here by the same steps in the same order, with the detector set +/// up the same way. Duplicated rather than trusted: a core that changed its +/// guess would have a report name one encoding while the text was read as +/// another, which is what `the_encoding_named_is_the_one_the_core_read` +/// catches on the next bump. +fn encoding_read_as(bytes: &[u8]) -> &'static Encoding { + if let Some((encoding, _)) = Encoding::for_bom(bytes) { + return encoding; } + if std::str::from_utf8(bytes).is_ok() { + return encoding_rs::UTF_8; + } + let mut detector = EncodingDetector::new(Iso2022JpDetection::Deny); + detector.feed(bytes, true); + detector.guess(None, Utf8Detection::Deny) } /// `#EXTINF:duration,display text`. @@ -246,7 +253,7 @@ mod tests { #[test] fn a_plain_file_is_one_target_per_line() { let file = parse(b"Artist/Album/01.flac\nArtist/Album/02.flac\n"); - assert_eq!(file.encoding, TextEncoding::Utf8); + assert_eq!(file.encoding, TextEncoding::UTF_8); assert_eq!(file.name, None); assert_eq!( targets(&file), @@ -291,7 +298,7 @@ mod tests { let mut bytes = vec![0xEF, 0xBB, 0xBF]; bytes.extend_from_slice(b"#EXTM3U\r\nArtist/Album/01.flac\r\n"); let file = parse(&bytes); - assert_eq!(file.encoding, TextEncoding::Utf8); + assert_eq!(file.encoding, TextEncoding::UTF_8); assert_eq!(targets(&file), ["Artist/Album/01.flac"]); } @@ -303,7 +310,7 @@ mod tests { bytes.push(0xF6); bytes.extend_from_slice(b"rk/Vespertine/01.flac\n"); let file = parse(&bytes); - assert_eq!(file.encoding, TextEncoding::Windows1252); + assert_eq!(file.encoding.name(), "windows-1252"); assert_eq!(targets(&file), ["Björk/Vespertine/01.flac"]); // And the thirty-two code points that tell Windows-1252 from Latin-1: @@ -318,6 +325,100 @@ mod tests { ); } + /// What Notepad saves as "Unicode", and what some players export: UTF-16 + /// little-endian behind its byte-order mark. Read as Windows-1252, every + /// other character was a NUL and no path matched anything. + #[test] + fn a_utf16_file_is_read_by_its_byte_order_mark() { + let text = "#EXTM3U\r\n#EXTINF:245,Björk - Hidden Place\r\nBjörk/Vespertine/01.flac\r\n"; + let mut bytes = vec![0xFF, 0xFE]; + bytes.extend(text.encode_utf16().flat_map(u16::to_le_bytes)); + let file = parse(&bytes); + assert_eq!(serde_json::to_value(file.encoding).unwrap(), "utf-16le"); + assert_eq!(targets(&file), ["Björk/Vespertine/01.flac"]); + assert_eq!( + file.entries[0].artist_and_title(), + Some(("Björk", "Hidden Place")) + ); + } + + /// A playlist written on a Japanese Windows, in Shift_JIS. The Windows-1252 + /// fallback read it without failing, which was worse: every name came back + /// as Latin nonsense that matched no path and no title. + #[test] + fn a_file_in_an_east_asian_code_page_is_read_in_it() { + let text = "#EXTM3U\r\n#PLAYLIST:夏の夜に聴きたい歌\r\n\ + #EXTINF:245,宇多田ヒカル - 花束を君に\r\n\ + 宇多田ヒカル/ファントーム/01 花束を君に.flac\r\n\ + #EXTINF:301,椎名林檎 - 丸の内サディスティック\r\n\ + 椎名林檎/無罪モラトリアム/03 丸の内サディスティック.flac\r\n"; + let (bytes, _, unmappable) = encoding_rs::SHIFT_JIS.encode(text); + assert!(!unmappable, "Shift_JIS holds the sample"); + assert!(std::str::from_utf8(&bytes).is_err()); + let file = parse(&bytes); + assert_eq!(serde_json::to_value(file.encoding).unwrap(), "shift_jis"); + assert_eq!(file.name.as_deref(), Some("夏の夜に聴きたい歌")); + assert_eq!( + targets(&file), + [ + "宇多田ヒカル/ファントーム/01 花束を君に.flac", + "椎名林檎/無罪モラトリアム/03 丸の内サディスティック.flac", + ] + ); + assert_eq!( + file.entries[1].artist_and_title(), + Some(("椎名林檎", "丸の内サディスティック")) + ); + } + + /// The report names the encoding the core read, not a second guess that + /// happens to agree today: decoding each sample with the named encoding + /// gives back the core's text exactly. + #[test] + fn the_encoding_named_is_the_one_the_core_read() { + let samples: [(&str, &'static Encoding); 6] = [ + ( + "#EXTINF:1,Björk - Cocoon\nBjörk/Vespertine/02.flac\n", + encoding_rs::WINDOWS_1252, + ), + ( + "#EXTINF:1,宇多田ヒカル - 花束を君に\n宇多田ヒカル/01 花束を君に.flac\n", + encoding_rs::SHIFT_JIS, + ), + ( + "#EXTINF:1,王菲 - 红豆\n王菲/唱游/03 红豆.flac\n", + encoding_rs::GB18030, + ), + ( + "#EXTINF:1,周杰倫 - 晴天\n周杰倫/葉惠美/01 晴天.flac\n", + encoding_rs::BIG5, + ), + ( + "#EXTINF:1,아이유 - 밤편지\n아이유/팔레트/01 밤편지.flac\n", + encoding_rs::EUC_KR, + ), + ( + "#EXTINF:1,Björk - Cocoon\nBjörk/Vespertine/02.flac\n", + encoding_rs::UTF_8, + ), + ]; + for (text, encoding) in samples { + let (bytes, _, unmappable) = encoding.encode(text); + assert!(!unmappable, "{} holds the sample", encoding.name()); + let named = encoding_read_as(&bytes); + assert_eq!( + named.decode_without_bom_handling(&bytes).0, + waveflow_core::text_encoding::decode(&bytes), + "{} was named {}", + encoding.name(), + named.name() + ); + } + let mut marked = vec![0xFE, 0xFF]; + marked.extend("Björk/01.flac".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(encoding_read_as(&marked), encoding_rs::UTF_16BE); + } + #[test] fn every_shape_of_target_reduces_to_the_same_segments() { let expected = ["Artist", "Album", "01.flac"]; diff --git a/src/services/mod.rs b/src/services/mod.rs index 2d44f8bf..e2cc2e79 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1067,8 +1067,11 @@ pub struct PlaylistImport { pub playlist: Option, /// The name used, from the request or from the file's own `#PLAYLIST`. pub name: String, - /// How the bytes were read as text. A file that announced itself as UTF-8 - /// and came back `windows-1252` has been read, not mangled in silence. + /// How the bytes were read as text, by the encoding's WHATWG name in lower + /// case: `utf-8`, `utf-16le`, `windows-1252`, `shift_jis`, `gbk`… A file + /// that announced itself as UTF-8 and came back anything else has been + /// read, not mangled in silence. + #[schema(value_type = String, example = "utf-8")] pub encoding: TextEncoding, /// Targets the file named, resolved or not. pub entries: usize, diff --git a/tests/native_api.rs b/tests/native_api.rs index 5de0f74d..73032790 100644 --- a/tests/native_api.rs +++ b/tests/native_api.rs @@ -3629,6 +3629,29 @@ async fn an_m3u_import_places_what_it_can_and_reports_the_rest() { "and the name did not become an entry of its own" ); + // **A UTF-16 file is read by its byte-order mark.** Notepad's "Unicode" + // and some players' exports write it. Read as Windows-1252, every other + // character came back a NUL and nothing in the file resolved. + let mut utf16 = vec![0xFF, 0xFE]; + utf16.extend( + "#EXTM3U\r\n#EXTINF:300,Björk - Hidden Place\r\nBjörk/Vespertine/01 Hidden Place.flac\r\n" + .encode_utf16() + .flat_map(u16::to_le_bytes), + ); + let (status, wide) = import( + "/api/v2/playlists/import?name=Wide&dry_run=true".into(), + utf16, + token.clone(), + ) + .await; + assert_eq!(status, StatusCode::OK, "{wide}"); + assert_eq!(wide["entries"], 1); + assert_eq!(wide["resolved"], 1, "the path it names is found: {wide}"); + assert_eq!( + wide["encoding"], "utf-16le", + "and the mark named the encoding" + ); + // A file nothing names is refused rather than stored namelessly. let (status, _body) = import( "/api/v2/playlists/import".into(), diff --git a/webapp/src/api/playlists.ts b/webapp/src/api/playlists.ts index 42b9888e..dbb80cb2 100644 --- a/webapp/src/api/playlists.ts +++ b/webapp/src/api/playlists.ts @@ -23,8 +23,12 @@ export type PlaylistImport = { /** The playlist as created; `null` for a dry run. */ playlist: Playlist | null; name: string; - /** How the server read the bytes: `.m3u` is frequently not UTF-8. */ - encoding: "utf-8" | "windows-1252"; + /** + * How the server read the bytes: `.m3u` is frequently not UTF-8. The + * encoding's WHATWG name in lower case — `utf-8`, `utf-16le`, + * `windows-1252`, `shift_jis`, `gbk`… — so an open set, not a union. + */ + encoding: string; /** Targets the file named, placed or not. */ entries: number; resolved: number; diff --git a/webapp/src/components/playlist-import.tsx b/webapp/src/components/playlist-import.tsx index 029217dc..9113457c 100644 --- a/webapp/src/components/playlist-import.tsx +++ b/webapp/src/components/playlist-import.tsx @@ -165,10 +165,10 @@ export function PlaylistImportPanel({ entries: preview.report.entries, })}

- {/* Said only when it is worth saying. A file read as Windows-1252 - was not UTF-8, and a wrong guess there mangles accented titles — - so the reader is told which guess was made rather than left to - wonder why a name looks odd. */} + {/* Said only when it is worth saying. A file read as anything but + UTF-8 had its encoding named by a byte-order mark or guessed, and + a wrong guess mangles every title — so the reader is told which + one was used rather than left to wonder why a name looks odd. */} {preview.report.encoding !== "utf-8" ? (

{t("import.encoding", { encoding: preview.report.encoding })} From 2c76a66ec43453d6f0446918049caf5805e9b963 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Wed, 21 Oct 2026 05:07:35 +0200 Subject: [PATCH 4/4] docs(api): say that a gb18030 playlist is reported as gbk chardetng never names GB18030: its README lists it as "detected as GBK", so a GB18030 `.m3u` comes back with `"encoding": "gbk"`. The text is read in full all the same, because encoding_rs's GBK decoder is its GB18030 decoder. Checked with a GB18030 file holding a four-byte character outside GBK (U+20000): reported `gbk`, and every character read back. Raised in review. Signed-off-by: InstaZDLL --- docs/api-v2-guide.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/api-v2-guide.md b/docs/api-v2-guide.md index 7ef65f9a..d0467e99 100644 --- a/docs/api-v2-guide.md +++ b/docs/api-v2-guide.md @@ -745,7 +745,9 @@ BE); otherwise the server reads UTF-8 when the bytes are UTF-8 and recognises the encoding from the bytes when they are not — GB18030, Big5, Shift_JIS, EUC-KR, Windows-1252 and the like — and **says which it used** in the response, as the encoding's WHATWG name in lower case (`utf-8`, `utf-16le`, `shift_jis`, -`windows-1252`…). +`windows-1252`…). A GB18030 file is reported as `gbk`: the detector never +names GB18030, and the GBK decoder is the GB18030 one, so the text is read in +full under either name. ```bash # What it would do, without doing it