diff --git a/dataspeech/gpu_enrichments/_brouhaha_compat.py b/dataspeech/gpu_enrichments/_brouhaha_compat.py new file mode 100644 index 00000000..67397e48 --- /dev/null +++ b/dataspeech/gpu_enrichments/_brouhaha_compat.py @@ -0,0 +1,84 @@ +"""Minimal stub of brouhaha model classes for checkpoint deserialization. + +The ylacombe/brouhaha-best checkpoint references brouhaha.models.CustomPyanNetModel. +PyTorch Lightning needs these classes importable to load the checkpoint. +This module provides just the model architecture definitions (no other brouhaha deps). +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from pyannote.audio import Model +from pyannote.audio.models.segmentation import PyanNet + +SNR_MIN = -15 +SNR_MAX = 80 +C50_MIN = -10 +C50_MAX = 60 + + +class ParametricSigmoid(nn.Module): + def __init__(self, alpha: float, beta: float) -> None: + super().__init__() + self.alpha = alpha + self.beta = beta + + def forward(self, x: torch.Tensor): + return (self.beta - self.alpha) * F.sigmoid(x) + self.alpha + + +class CustomClassifier(nn.Module): + def __init__(self, in_features, out_features: int) -> None: + super().__init__() + self.linears = nn.ModuleDict({ + 'vad': nn.Linear(in_features, out_features), + 'snr': nn.Linear(in_features, 1), + 'c50': nn.Linear(in_features, 1), + }) + + def forward(self, x: torch.Tensor): + out = dict() + for mode, linear in self.linears.items(): + _output = linear(x) + out[mode] = _output + return out + + +class CustomActivation(nn.Module): + def __init__(self) -> None: + super().__init__() + self.activations = nn.ModuleDict({ + 'vad': nn.Sigmoid(), + 'snr': ParametricSigmoid(SNR_MAX, SNR_MIN), + 'c50': ParametricSigmoid(C50_MAX, C50_MIN), + }) + + def forward(self, x: torch.Tensor): + out = list() + for mode, activation in self.activations.items(): + _output = activation(x[mode]) + out.append(_output) + out = torch.stack(out) + out = rearrange(out, "n b t o -> b t (n o)") + return out + + +class RegressiveSegmentationModelMixin(Model): + def build(self): + nb_classif = len(set(self.specifications.classes) - set(['snr', 'c50'])) + self.classifier = CustomClassifier(32 * 2, nb_classif) + self.activation = CustomActivation() + + +class CustomPyanNetModel(RegressiveSegmentationModelMixin, PyanNet): + def build(self): + if self.hparams.linear["num_layers"] > 0: + in_features = self.hparams.linear["hidden_size"] + else: + in_features = self.hparams.lstm["hidden_size"] * ( + 2 if self.hparams.lstm["bidirectional"] else 1 + ) + nb_classif = len(set(self.specifications.classes) - set(['snr', 'c50'])) + self.classifier = CustomClassifier(in_features, nb_classif) + self.activation = CustomActivation() diff --git a/dataspeech/gpu_enrichments/_torbi_compat.py b/dataspeech/gpu_enrichments/_torbi_compat.py new file mode 100644 index 00000000..6478d8ea --- /dev/null +++ b/dataspeech/gpu_enrichments/_torbi_compat.py @@ -0,0 +1,90 @@ +"""Compatibility shim for torbi on unsupported torch/CUDA versions. + +torbi ships prebuilt C++ binaries for specific torch+CUDA combinations. +When no matching binary exists (e.g. torch 2.9+), import fails with +FileNotFoundError. This module patches torbi to fall back to a pure-Python +Viterbi implementation using librosa.sequence.viterbi — the same algorithm +used by torbi's own reference implementation (torbi.reference.core). +""" + +import sys + + +def ensure_torbi(): + """Ensure torbi is importable, patching with a librosa fallback if needed.""" + if 'torbi' in sys.modules: + return + + try: + import torbi # noqa: F401 + return + except FileNotFoundError: + pass + + # torbi's C++ binary is unavailable for this torch/CUDA version. + # Create a minimal stub module providing only from_probabilities(), + # which is the sole function penn uses from torbi. + import types + + import numpy as np + import torch + + torbi_mod = types.ModuleType('torbi') + torbi_mod.__package__ = 'torbi' + sys.modules['torbi'] = torbi_mod + + def from_probabilities( + observation, + batch_frames=None, + transition=None, + initial=None, + log_probs=False, + gpu=None, + num_threads=1, + ): + """Pure-Python Viterbi decoding via librosa (fallback for missing C++ binary). + + Replicates the interface of torbi.core.from_probabilities but uses + librosa.sequence.viterbi for the actual decoding, matching the + algorithm in torbi.reference.core.from_probabilities. + """ + import librosa + + device = observation.device + batch, frames, states = observation.shape + + # Convert to probability space for librosa + obs_probs = torch.exp(observation) if log_probs else observation + obs_np = obs_probs.to(torch.float32).cpu().numpy() + + # Initial distribution (librosa expects probabilities, not log-probs) + if initial is None: + initial_np = np.full((states,), 1.0 / states, dtype=np.float32) + else: + init_t = torch.exp(initial) if log_probs else initial + initial_np = init_t.to(torch.float32).cpu().numpy() + + # Transition matrix (librosa expects probabilities) + if transition is None: + trans_np = np.full( + (states, states), 1.0 / states, dtype=np.float32) + else: + trans_t = torch.exp(transition) if log_probs else transition + trans_np = trans_t.to(torch.float32).cpu().numpy() + + # Decode each batch item + results = [] + for i in range(batch): + n = batch_frames[i].item() if batch_frames is not None else frames + indices = librosa.sequence.viterbi( + obs_np[i, :n].T, trans_np, p_init=initial_np) + if n < frames: + padded = np.zeros(frames, dtype=np.int32) + padded[:n] = indices + indices = padded + results.append(torch.tensor( + indices.astype(np.int32), dtype=torch.int, device=device)) + + return torch.stack(results) + + torbi_mod.from_probabilities = from_probabilities diff --git a/dataspeech/gpu_enrichments/pitch.py b/dataspeech/gpu_enrichments/pitch.py index 16513ab7..02dcd1a5 100644 --- a/dataspeech/gpu_enrichments/pitch.py +++ b/dataspeech/gpu_enrichments/pitch.py @@ -1,4 +1,6 @@ -import torch +import torch +from ._torbi_compat import ensure_torbi +ensure_torbi() import penn diff --git a/dataspeech/gpu_enrichments/snr_and_reverb.py b/dataspeech/gpu_enrichments/snr_and_reverb.py index d5c9d381..adabe1da 100644 --- a/dataspeech/gpu_enrichments/snr_and_reverb.py +++ b/dataspeech/gpu_enrichments/snr_and_reverb.py @@ -1,73 +1,269 @@ -from pyannote.audio import Model -from pathlib import Path -from brouhaha.pipeline import RegressiveActivityDetectionPipeline -import torch -from huggingface_hub import hf_hub_download +from typing import Callable, List, Optional, Tuple, Union + import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange +from huggingface_hub import hf_hub_download +from pathlib import Path + +import warnings +with warnings.catch_warnings(): + # Newer pyannote.audio (3.3+) warns about torchcodec at import time. + # torchcodec is not needed here — we pass pre-loaded waveforms, not files. + warnings.filterwarnings("ignore", message="torchcodec", category=UserWarning) + from pyannote.audio import Model, Inference + from pyannote.audio.core.model import Specifications + from pyannote.audio.core.task import Resolution + from pyannote.audio.utils.multi_task import map_with_specifications + from pyannote.audio.utils.signal import Binarize + from pyannote.core import Segment, SlidingWindow, SlidingWindowFeature + + +class _BrouhahaInference(Inference): + """Inference subclass that uses repeat-based padding for the last chunk, + matching the original brouhaha-vad BrouhahaInference behavior exactly.""" + + def slide( + self, + waveform: torch.Tensor, + sample_rate: int, + hook: Optional[Callable], + ) -> Union[SlidingWindowFeature, Tuple[SlidingWindowFeature]]: + + window_size: int = self.model.audio.get_num_samples(self.duration) + step_size: int = round(self.step * sample_rate) + _, num_samples = waveform.shape + + def __frames( + receptive_field, specifications: Optional[Specifications] = None + ) -> SlidingWindow: + if specifications.resolution == Resolution.CHUNK: + return SlidingWindow(start=0.0, duration=self.duration, step=self.step) + return receptive_field + + frames: Union[SlidingWindow, Tuple[SlidingWindow]] = map_with_specifications( + self.model.specifications, __frames, self.model.receptive_field + ) + + # prepare complete chunks + if num_samples >= window_size: + chunks: torch.Tensor = rearrange( + waveform.unfold(1, window_size, step_size), + "channel chunk frame -> chunk channel frame", + ) + num_chunks, _, _ = chunks.shape + else: + num_chunks = 0 + + # prepare last incomplete chunk + has_last_chunk = (num_samples < window_size) or ( + num_samples - window_size + ) % step_size > 0 + + if has_last_chunk: + # repeat last chunk to fill window (brouhaha-style, not zero-pad) + last_chunk: torch.Tensor = waveform[:, num_chunks * step_size:] + channel, last_window_size = last_chunk.shape + num_repeat = window_size // last_window_size + 1 + last_chunk = last_chunk.repeat((channel, num_repeat)) + last_chunk = last_chunk[:, :window_size] + + def __empty_list(**kwargs): + return list() + + outputs: Union[ + List[np.ndarray], Tuple[List[np.ndarray]] + ] = map_with_specifications(self.model.specifications, __empty_list) + + if hook is not None: + hook(completed=0, total=num_chunks + has_last_chunk) + + def __append_batch(output, batch_output, **kwargs) -> None: + output.append(batch_output) + return + + # slide over audio chunks in batch + for c in np.arange(0, num_chunks, self.batch_size): + batch: torch.Tensor = chunks[c : c + self.batch_size] + + batch_outputs: Union[np.ndarray, Tuple[np.ndarray]] = self.infer(batch) + + _ = map_with_specifications( + self.model.specifications, __append_batch, outputs, batch_outputs + ) + + if hook is not None: + hook(completed=c + self.batch_size, total=num_chunks + has_last_chunk) + + # process orphan last chunk + if has_last_chunk: + last_outputs = self.infer(last_chunk[None]) + + _ = map_with_specifications( + self.model.specifications, __append_batch, outputs, last_outputs + ) + + if hook is not None: + hook( + completed=num_chunks + has_last_chunk, + total=num_chunks + has_last_chunk, + ) + + def __vstack(output: List[np.ndarray], **kwargs) -> np.ndarray: + return np.vstack(output) + + outputs: Union[np.ndarray, Tuple[np.ndarray]] = map_with_specifications( + self.model.specifications, __vstack, outputs + ) + + def __aggregate( + outputs: np.ndarray, + frames: SlidingWindow, + specifications: Optional[Specifications] = None, + ) -> SlidingWindowFeature: + if ( + self.skip_aggregation + or specifications.resolution == Resolution.CHUNK + or ( + specifications.permutation_invariant + and self.pre_aggregation_hook is None + ) + ): + frames = SlidingWindow( + start=0.0, duration=self.duration, step=self.step + ) + return SlidingWindowFeature(outputs, frames) + + if self.pre_aggregation_hook is not None: + outputs = self.pre_aggregation_hook(outputs) + + aggregated = self.aggregate( + SlidingWindowFeature( + outputs, + SlidingWindow(start=0.0, duration=self.duration, step=self.step), + ), + frames, + warm_up=self.warm_up, + hamming=True, + missing=0.0, + ) + + # remove padding that was added to last chunk + if has_last_chunk: + aggregated.data = aggregated.crop( + Segment(0.0, num_samples / sample_rate), mode="loose" + ) + + return aggregated + + return map_with_specifications( + self.model.specifications, __aggregate, outputs, frames + ) + model = None ratio = 16000/270 +# Binarize with brouhaha's default parameters (onset=offset=0.780, no min duration) +_binarize = Binarize(onset=0.780, offset=0.780) + + +def _run_inference(sample, inference): + """Run inference on a single audio sample and return annotation, snr, c50.""" + segmentations = inference({"sample_rate": sample["sampling_rate"], + "waveform": torch.tensor(sample["array"][None, :]).to(inference.device).float()}) + + # Extract VAD column and binarize (replicates RegressiveActivityDetectionPipeline.apply) + vad_scores = SlidingWindowFeature( + np.expand_dims(segmentations.data[:, 0], axis=1), + segmentations.sliding_window, + ) + annotation = _binarize(vad_scores) + + snr_array = segmentations.data[:, 1] + c50_array = segmentations.data[:, 2] + + return annotation, snr_array, c50_array + + def snr_apply(batch, rank=None, audio_column_name="audio", batch_size=32): global model if model is None: - model = Model.from_pretrained( - Path(hf_hub_download(repo_id="ylacombe/brouhaha-best", filename="best.ckpt")), - strict=False, - ) + import sys + from dataspeech.gpu_enrichments import _brouhaha_compat + import types + + # Register stub so that the checkpoint's reference to brouhaha.models + # resolves without installing the full brouhaha-vad package. + if "brouhaha" not in sys.modules: + brouhaha_pkg = types.ModuleType("brouhaha") + brouhaha_pkg.__path__ = [] + sys.modules["brouhaha"] = brouhaha_pkg + if "brouhaha.models" not in sys.modules: + sys.modules["brouhaha.models"] = _brouhaha_compat + + # The ylacombe/brouhaha-best checkpoint contains pickle-serialized objects + # (e.g. TorchVersion) that fail with torch.load's weights_only=True default + # in PyTorch 2.6+. Temporarily patch torch.load during model loading. + _original_torch_load = torch.load + def _patched_load(*args, **kwargs): + kwargs["weights_only"] = False + return _original_torch_load(*args, **kwargs) + torch.load = _patched_load + try: + model = Model.from_pretrained( + Path(hf_hub_download(repo_id="ylacombe/brouhaha-best", filename="best.ckpt")), + strict=False, + ) + finally: + torch.load = _original_torch_load if rank is not None or torch.cuda.device_count() > 0: # move the model to the right GPU if not there already device = f"cuda:{(rank or 0)% torch.cuda.device_count()}" - # move to device and create pipeline here because the pipeline moves to the first GPU it finds anyway model.to(device) - pipeline = RegressiveActivityDetectionPipeline(segmentation=model, batch_size = batch_size) - if rank: - pipeline.to(torch.device(device)) - - device = pipeline._models["segmentation"].device + inference = _BrouhahaInference(model, device=torch.device(model.device)) - if isinstance(batch[audio_column_name], list): + if isinstance(batch[audio_column_name], list): snr = [] c50 = [] vad_durations = [] for sample in batch[audio_column_name]: - res = pipeline({"sample_rate": sample["sampling_rate"], - "waveform": torch.tensor(sample["array"][None, :]).to(device).float()}) - - mask = np.full(res["snr"].shape, False) - for (segment, _) in res["annotation"].itertracks(): + annotation, snr_array, c50_array = _run_inference(sample, inference) + + mask = np.full(snr_array.shape, False) + for (segment, _) in annotation.itertracks(): start = int(segment.start * ratio) end = int(segment.end * ratio) mask[start:end] = True - mask = (~((res["snr"] == 0.0) & (res["c50"] == 0.0)) & mask) + mask = (~((snr_array == 0.0) & (c50_array == 0.0)) & mask) - vad_duration = sum(map(lambda x: x[0].duration, res["annotation"].itertracks())) - - snr.append(res["snr"][mask].mean()) - c50.append(res["c50"][mask].mean()) + vad_duration = sum(map(lambda x: x[0].duration, annotation.itertracks())) + + snr.append(snr_array[mask].mean()) + c50.append(c50_array[mask].mean()) vad_durations.append(np.float32(vad_duration)) - + # 16ms window batch["snr"] = snr batch["c50"] = c50 batch["speech_duration"] = vad_durations - + else: - res = pipeline({"sample_rate": batch[audio_column_name]["sampling_rate"], - "waveform": torch.tensor(batch[audio_column_name]["array"][None, :]).to(device).float()}) - - mask = np.full(res["snr"].shape, False) - for (segment, _) in res["annotation"].itertracks(): + annotation, snr_array, c50_array = _run_inference(batch[audio_column_name], inference) + + mask = np.full(snr_array.shape, False) + for (segment, _) in annotation.itertracks(): start = int(segment.start * ratio) end = int(segment.end * ratio) mask[start:end] = True - mask = (~((res["snr"] == 0.0) & (res["c50"] == 0.0)) & mask) + mask = (~((snr_array == 0.0) & (c50_array == 0.0)) & mask) + + vad_duration = sum(map(lambda x: x[0].duration, annotation.itertracks())) - vad_duration = sum(map(lambda x: x[0].duration, res["annotation"].itertracks())) - - batch["snr"] = res["snr"][mask].mean() - batch["c50"] = res["c50"][mask].mean() + batch["snr"] = snr_array[mask].mean() + batch["c50"] = c50_array[mask].mean() batch["speech_duration"] = vad_duration - - return batch \ No newline at end of file + + return batch diff --git a/requirements.txt b/requirements.txt index 93e2b8e4..8cb87704 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -datasets[audio] -https://github.com/marianne-m/brouhaha-vad/archive/main.zip +datasets[audio]==3.2.0 penn g2p demucs