feat: update output format to metadata.jsonl

CLAUDE.md

@@ -93,13 +93,14 @@ Uses `datasets` with `Audio(decode=False)` + manual `soundfile` decoding — avo
 
 ## Output Format
 
-`manifest.jsonl` — one JSON object per generated file:
+`metadata.jsonl` — one JSON object per generated file, following the HuggingFace [AudioFolder](https://huggingface.co/docs/datasets/audio_dataset#audiofolder) convention so the output directory is directly loadable with `datasets.load_dataset("audiofolder", data_dir="./out")`.
 
 ```json
 {
-  "audio": "common_voice_en_23136613_telecommunication.wav",
+  "file_name": "audio/common_voice_en_23136613_telecommunication.wav",
   "source": "common_voice_en_23136613.mp3",
   "dataset": "google/fleurs",
+  "language": "en-US",
   "preset": "telecommunication",
   "transcript": "...",
   "snr_db": 1.8,
@@ -127,7 +128,7 @@ uv run noisekit generate \
   --config en_us --split test \
   --samples 3 --presets clean_reference telecommunication bad_audio_encoding \
   --output ./test_out --seed 42
-cat test_out/manifest.jsonl
+cat test_out/metadata.jsonl
 
 # noisy_environment — auto-downloads MUSAN noise-only clips on first run
 uv run noisekit generate \

README.md

@@ -25,7 +25,7 @@ flowchart LR
     B --> D["bad_audio_encoding\n16-32 kbps MP3"]
     B --> E["noisy_environment\nReal ambient noise"]
     B --> F["clean_reference\nControl"]
-    C & D & E & F --> G[("WAVs +\nmanifest.jsonl\nPESQ · SNR · NISQA")]
+    C & D & E & F --> G[("WAVs +\nmetadata.jsonl\nPESQ · SNR · NISQA")]
 ```
 
 ## Install
@@ -74,20 +74,30 @@ Output:
 
 ```
 benchmark_dataset/
-├── manifest.jsonl          # one entry per generated file
+├── metadata.jsonl          # one entry per generated file (AudioFolder format)
 └── audio/
     ├── sample_0000_telecommunication.wav
     ├── sample_0001_bad_audio_encoding.wav
     └── ...
 ```
 
-Each manifest entry:
+The output is directly loadable as a HuggingFace dataset:
+
+```python
+from datasets import load_dataset
+ds = load_dataset("audiofolder", data_dir="./benchmark_dataset")
+```
+
+Each `metadata.jsonl` entry:
 
 ```json
 {
-  "audio": "sample_0042_telecommunication.wav",
-  "transcript": "the cat sat on the mat",
+  "file_name": "audio/sample_0042_telecommunication.wav",
+  "source": "common_voice_en_23136613.mp3",
+  "dataset": "google/fleurs",
+  "language": "en-US",
   "preset": "telecommunication",
+  "transcript": "the cat sat on the mat",
   "snr_db": 5.2,
   "pesq_mos": 2.78,
   "nisqa_mos": 2.14,

noisekit/dataset.py

@@ -26,6 +26,34 @@ def load_samples(
     return list(itertools.islice(ds, n))
 
 
+def _config_to_bcp47(config: str) -> str:
+    """Convert a HuggingFace config name to a BCP-47 tag.
+
+    ``en_us`` → ``en-US``, ``fr_fr`` → ``fr-FR``, ``en`` → ``en``.
+    """
+    parts = config.split("_", 1)
+    if len(parts) == 2:
+        return f"{parts[0].lower()}-{parts[1].upper()}"
+    return config.lower()
+
+
+def extract_language(sample: dict, config: str | None = None) -> str | None:
+    """Return a BCP-47 language tag for the sample.
+
+    Priority:
+    1. ``locale`` — already BCP-47 (Common Voice, Mozilla datasets).
+    2. ``config`` — HuggingFace subset name normalized to BCP-47
+       (e.g. ``en_us`` → ``en-US``). The per-sample ``language`` column is
+       intentionally skipped because datasets like FLEURS store full names
+       (``"English"``) which are not valid BCP-47 tags.
+    """
+    if locale := sample.get("locale"):
+        return locale
+    if config:
+        return _config_to_bcp47(config)
+    return None
+
+
 def extract_audio_and_text(sample: dict) -> tuple[np.ndarray, int, str]:
     audio_field = sample["audio"]
 

noisekit/pipeline.py

@@ -10,7 +10,7 @@
 from rich.console import Console
 from rich.progress import track
 
-from .dataset import extract_audio_and_text, load_samples
+from .dataset import extract_audio_and_text, extract_language, load_samples
 from .noise_cache import ensure_default_noise_dir
 from .scoring import _NISQA_KEYS, audio_stats, compute_nisqa, compute_pesq, compute_snr_db
 from .transforms import list_builtin_presets, load_preset, preset_requires_noise_dir
@@ -56,6 +56,7 @@ def run_generate(
 
     for i, sample in enumerate(track(raw_samples, description="Generating …")):
         ref_array, ref_sr, transcript = extract_audio_and_text(sample)
+        language = extract_language(sample, config)
         ref_16k = _resample_to_16k(ref_array, ref_sr)
 
         raw_path = sample.get("audio", {}).get("path") or ""
@@ -98,9 +99,10 @@ def run_generate(
 
             nisqa_scores = compute_nisqa(deg, 16000) if nisqa else dict.fromkeys(_NISQA_KEYS)
             entry: dict = {
-                "audio": filename,
+                "file_name": f"audio/{filename}",
                 "source": source_filename,
                 "dataset": dataset,
+                "language": language,
                 "preset": preset_name,
                 "transcript": transcript,
                 "snr_db": round(snr, 3),
@@ -109,13 +111,14 @@ def run_generate(
             }
             manifest.append(entry)
 
-    manifest_path = output_dir / "manifest.jsonl"
-    with open(manifest_path, "w", encoding="utf-8") as f:
+    metadata_path = output_dir / "metadata.jsonl"
+    with open(metadata_path, "w", encoding="utf-8") as f:
         for entry in manifest:
             f.write(json.dumps(entry) + "\n")
 
     console.print(f"\n[green]Done.[/green] {len(manifest)} files written to [bold]{output_dir}[/bold]")
-    console.print(f"Manifest: [bold]{manifest_path}[/bold]")
+    console.print(f"Metadata: [bold]{metadata_path}[/bold]")
+    console.print('[dim]Load with: datasets.load_dataset("audiofolder", data_dir="{output_dir}")[/dim]')
 
 
 def run_score(