From a22da70cb069f04ed9802609df8a507473de3713 Mon Sep 17 00:00:00 2001 From: karamouche Date: Fri, 22 May 2026 09:56:51 -0400 Subject: [PATCH] feat: update output format to metadata.jsonl --- CLAUDE.md | 7 ++++--- README.md | 20 +++++++++++++++----- noisekit/dataset.py | 28 ++++++++++++++++++++++++++++ noisekit/pipeline.py | 13 ++++++++----- 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b2a1da9..6273da4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,13 +93,14 @@ Uses `datasets` with `Audio(decode=False)` + manual `soundfile` decoding — avo ## Output Format -`manifest.jsonl` — one JSON object per generated file: +`metadata.jsonl` — one JSON object per generated file, following the HuggingFace [AudioFolder](https://huggingface.co/docs/datasets/audio_dataset#audiofolder) convention so the output directory is directly loadable with `datasets.load_dataset("audiofolder", data_dir="./out")`. ```json { - "audio": "common_voice_en_23136613_telecommunication.wav", + "file_name": "audio/common_voice_en_23136613_telecommunication.wav", "source": "common_voice_en_23136613.mp3", "dataset": "google/fleurs", + "language": "en-US", "preset": "telecommunication", "transcript": "...", "snr_db": 1.8, @@ -127,7 +128,7 @@ uv run noisekit generate \ --config en_us --split test \ --samples 3 --presets clean_reference telecommunication bad_audio_encoding \ --output ./test_out --seed 42 -cat test_out/manifest.jsonl +cat test_out/metadata.jsonl # noisy_environment — auto-downloads MUSAN noise-only clips on first run uv run noisekit generate \ diff --git a/README.md b/README.md index 64e1434..7a2c4f0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ flowchart LR B --> D["bad_audio_encoding\n16-32 kbps MP3"] B --> E["noisy_environment\nReal ambient noise"] B --> F["clean_reference\nControl"] - C & D & E & F --> G[("WAVs +\nmanifest.jsonl\nPESQ · SNR · NISQA")] + C & D & E & F --> G[("WAVs +\nmetadata.jsonl\nPESQ · SNR · NISQA")] ``` ## Install @@ -74,20 +74,30 @@ Output: ``` benchmark_dataset/ -├── manifest.jsonl # one entry per generated file +├── metadata.jsonl # one entry per generated file (AudioFolder format) └── audio/ ├── sample_0000_telecommunication.wav ├── sample_0001_bad_audio_encoding.wav └── ... ``` -Each manifest entry: +The output is directly loadable as a HuggingFace dataset: + +```python +from datasets import load_dataset +ds = load_dataset("audiofolder", data_dir="./benchmark_dataset") +``` + +Each `metadata.jsonl` entry: ```json { - "audio": "sample_0042_telecommunication.wav", - "transcript": "the cat sat on the mat", + "file_name": "audio/sample_0042_telecommunication.wav", + "source": "common_voice_en_23136613.mp3", + "dataset": "google/fleurs", + "language": "en-US", "preset": "telecommunication", + "transcript": "the cat sat on the mat", "snr_db": 5.2, "pesq_mos": 2.78, "nisqa_mos": 2.14, diff --git a/noisekit/dataset.py b/noisekit/dataset.py index f88e6dd..edeb7a6 100644 --- a/noisekit/dataset.py +++ b/noisekit/dataset.py @@ -26,6 +26,34 @@ def load_samples( return list(itertools.islice(ds, n)) +def _config_to_bcp47(config: str) -> str: + """Convert a HuggingFace config name to a BCP-47 tag. + + ``en_us`` → ``en-US``, ``fr_fr`` → ``fr-FR``, ``en`` → ``en``. + """ + parts = config.split("_", 1) + if len(parts) == 2: + return f"{parts[0].lower()}-{parts[1].upper()}" + return config.lower() + + +def extract_language(sample: dict, config: str | None = None) -> str | None: + """Return a BCP-47 language tag for the sample. + + Priority: + 1. ``locale`` — already BCP-47 (Common Voice, Mozilla datasets). + 2. ``config`` — HuggingFace subset name normalized to BCP-47 + (e.g. ``en_us`` → ``en-US``). The per-sample ``language`` column is + intentionally skipped because datasets like FLEURS store full names + (``"English"``) which are not valid BCP-47 tags. + """ + if locale := sample.get("locale"): + return locale + if config: + return _config_to_bcp47(config) + return None + + def extract_audio_and_text(sample: dict) -> tuple[np.ndarray, int, str]: audio_field = sample["audio"] diff --git a/noisekit/pipeline.py b/noisekit/pipeline.py index 13db8bc..a91040c 100644 --- a/noisekit/pipeline.py +++ b/noisekit/pipeline.py @@ -10,7 +10,7 @@ from rich.console import Console from rich.progress import track -from .dataset import extract_audio_and_text, load_samples +from .dataset import extract_audio_and_text, extract_language, load_samples from .noise_cache import ensure_default_noise_dir from .scoring import _NISQA_KEYS, audio_stats, compute_nisqa, compute_pesq, compute_snr_db from .transforms import list_builtin_presets, load_preset, preset_requires_noise_dir @@ -56,6 +56,7 @@ def run_generate( for i, sample in enumerate(track(raw_samples, description="Generating …")): ref_array, ref_sr, transcript = extract_audio_and_text(sample) + language = extract_language(sample, config) ref_16k = _resample_to_16k(ref_array, ref_sr) raw_path = sample.get("audio", {}).get("path") or "" @@ -98,9 +99,10 @@ def run_generate( nisqa_scores = compute_nisqa(deg, 16000) if nisqa else dict.fromkeys(_NISQA_KEYS) entry: dict = { - "audio": filename, + "file_name": f"audio/{filename}", "source": source_filename, "dataset": dataset, + "language": language, "preset": preset_name, "transcript": transcript, "snr_db": round(snr, 3), @@ -109,13 +111,14 @@ def run_generate( } manifest.append(entry) - manifest_path = output_dir / "manifest.jsonl" - with open(manifest_path, "w", encoding="utf-8") as f: + metadata_path = output_dir / "metadata.jsonl" + with open(metadata_path, "w", encoding="utf-8") as f: for entry in manifest: f.write(json.dumps(entry) + "\n") console.print(f"\n[green]Done.[/green] {len(manifest)} files written to [bold]{output_dir}[/bold]") - console.print(f"Manifest: [bold]{manifest_path}[/bold]") + console.print(f"Metadata: [bold]{metadata_path}[/bold]") + console.print('[dim]Load with: datasets.load_dataset("audiofolder", data_dir="{output_dir}")[/dim]') def run_score(