Remove hard torchaudio dependency for inference; add kaldi-native-fbank fbank backend (#3526)

* Remove hard torchaudio dependency for inference; add kaldi-native-fbank fbank backend

Make torchaudio an optional dependency for the inference path so FunASR
can run on Ascend NPU (aarch64) servers, where no matching torchaudio
wheel exists (Huawei's Ascend torch is version-decoupled from the PyTorch
upstream release line).

- Add funasr/utils/fbank.py: an fbank shim that delegates to
  torchaudio.compliance.kaldi when torchaudio is installed, otherwise
  falls back to kaldi-native-fbank (a torch-free C++ library with aarch64
  wheels). Options and frame-shift API are mapped to torchaudio semantics
  so outputs are equivalent.
- Switch funasr/frontends/wav_frontend.py, funasr/utils/speaker_utils.py
  and funasr/models/campplus/utils.py to the shim.
- Make the torchaudio import optional in funasr/utils/load_utils.py
  (resampler falls back to librosa).

No device-specific branching is introduced — a pure backend fallback that
also benefits CPU-only and other accelerator platforms.

Verified end-to-end on Ascend 910B (npu:0) with torchaudio absent:
paraformer-large loads and infers correctly, RTF ~0.016 (~60x realtime).

* Make torchaudio import optional in paraformer_v2, fun_asr_nano and dataset preprocessors

Follow-up to the inference-path change: make the remaining top-level
torchaudio imports optional so these modules import cleanly when
torchaudio is not installed (e.g. on Ascend NPU servers).

- funasr/models/paraformer_v2_community/model.py
- funasr/models/fun_asr_nano/tools/utils.py
- funasr/datasets/audio_datasets/preprocessor.py
- funasr/datasets/llm_datasets/preprocessor.py

Functions that still require torchaudio (forced_align, Resample) degrade
to a clear error only when called, rather than a hard import failure.

* Address review: declare knf extra, harden fbank fallback and torchaudio guards

Respond to maintainer review feedback.

- Declare kaldi-native-fbank as an optional 'knf' extra in setup.py; the
  fbank module now raises an actionable ImportError only when fbank() is
  actually called with neither torchaudio nor kaldi-native-fbank installed
  (previously it failed at import time with a bare ModuleNotFoundError).
- Harden the fbank fallback contract: handle the channel argument explicitly
  and reject torchaudio-only options (subtract_mean, min_duration, VTLN
  warping, non-default blackman_coeff) with NotImplementedError instead of
  silently dropping them via **kwargs.
- Add a shared guard funasr/utils/torchaudio_compat.py and use it for
  resampling, forced alignment and speed perturbation so a missing torchaudio
  fails loudly instead of returning a silently wrong result (empty alignment).
- Add regression tests (tests/test_torchaudio_optional.py) that import the
  affected modules with torchaudio absent.

Author: li-lizhe

funasr/datasets/audio_datasets/preprocessor.py

@@ -7,12 +7,16 @@
 import torch.distributed as dist
 from typing import Collection
 import torch
-import torchaudio
+try:
+    import torchaudio
+except ImportError:
+    torchaudio = None
 from torch import nn
 import random
 import re
 from funasr.tokenizer.cleaner import TextCleaner
 from funasr.register import tables
+from funasr.utils.torchaudio_compat import require_torchaudio
 
 
 @tables.register("preprocessor_classes", "SpeechPreprocessSpeedPerturb")
@@ -39,6 +43,7 @@ def forward(self, waveform, fs, **kwargs):
             return waveform
         speed = random.choice(self.speed_perturb)
         if speed != 1.0:
+            torchaudio = require_torchaudio("speed perturbation")
             if not isinstance(waveform, torch.Tensor):
                 waveform = torch.tensor(waveform)
             waveform, _ = torchaudio.sox_effects.apply_effects_tensor(
@@ -79,4 +84,4 @@ def forward(self, text, **kwargs):
             """
         text = self.text_cleaner(text)
 
-        return text
+        return text

funasr/datasets/llm_datasets/preprocessor.py

@@ -7,7 +7,10 @@
 import torch.distributed as dist
 from typing import Collection
 import torch
-import torchaudio
+try:
+    import torchaudio
+except ImportError:
+    torchaudio = None
 from torch import nn
 import random
 import re

funasr/frontends/wav_frontend.py

@@ -5,7 +5,7 @@
 import numpy as np
 import torch
 import torch.nn as nn
-import torchaudio.compliance.kaldi as kaldi
+from funasr.utils import fbank as kaldi
 from torch.nn.utils.rnn import pad_sequence
 
 import funasr.frontends.eend_ola_feature as eend_ola_feature

funasr/models/campplus/utils.py

@@ -16,7 +16,7 @@
 from pathlib import Path
 from typing import Generator, Union
 from abc import ABCMeta, abstractmethod
-import torchaudio.compliance.kaldi as Kaldi
+from funasr.utils import fbank as Kaldi
 
 from funasr.models.transformer.utils.nets_utils import pad_list
 

funasr/models/fun_asr_nano/tools/utils.py

@@ -1,14 +1,15 @@
 from itertools import groupby
 
+import librosa
 import soundfile as sf
 import torch
-import torchaudio
-import torchaudio.functional as F
+
+from funasr.utils.torchaudio_compat import require_torchaudio, torchaudio_available
 
 
 def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):
     """Load audio.
-    
+
         Args:
             wav_path: TODO.
             rate: TODO.
@@ -29,27 +30,34 @@ def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float =
             audio_tensor = audio_tensor.unsqueeze(0)
         else:
             audio_tensor = audio_tensor.T
-        resampler = torchaudio.transforms.Resample(orig_freq=f.samplerate, new_freq=rate)
-        audio_tensor = resampler(audio_tensor)
+        if torchaudio_available():
+            torchaudio = require_torchaudio("resampling")
+            resampler = torchaudio.transforms.Resample(orig_freq=f.samplerate, new_freq=rate)
+            audio_tensor = resampler(audio_tensor)
+        else:
+            audio_tensor = torch.from_numpy(
+                librosa.resample(audio_tensor.numpy(), orig_sr=f.samplerate, target_sr=rate)
+            )
         if audio_tensor.shape[0] == 1:
             audio_tensor = audio_tensor.squeeze(0)
     return audio_tensor, rate if rate is not None else f.samplerate
 
 
 def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):
     """Forced align.
-    
+
         Args:
             log_probs: TODO.
             targets: TODO.
             blank: TODO.
         """
+    torchaudio = require_torchaudio("forced_align")
     items = []
     try:
         # The current version only supports batch_size==1.
         log_probs, targets = log_probs.unsqueeze(0).cpu(), targets.unsqueeze(0).cpu()
         assert log_probs.shape[1] >= targets.shape[1]
-        alignments, scores = F.forced_align(log_probs, targets, blank=blank)
+        alignments, scores = torchaudio.functional.forced_align(log_probs, targets, blank=blank)
         alignments, scores = alignments[0], torch.exp(scores[0]).tolist()
         # use enumerate to keep track of the original indices, then group by token value
         for token, group in groupby(enumerate(alignments), key=lambda item: item[1]):
@@ -67,6 +75,6 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
                     "score": round(score, 3),
                 }
             )
-    except:
+    except Exception:
         pass
-    return items
+    return items

funasr/models/paraformer_v2_community/model.py

@@ -24,8 +24,12 @@
 from funasr.models.transformer.utils.nets_utils import make_pad_mask
 from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
 from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
+from funasr.utils.torchaudio_compat import require_torchaudio
 from torch.nn.utils.rnn import pad_sequence
-import torchaudio
+try:
+    import torchaudio
+except ImportError:
+    torchaudio = None
 
 @tables.register("model_classes", "Paraformer_v2_community")
 class Paraformer(torch.nn.Module):
@@ -406,6 +410,7 @@ def force_align(self, ctc_probs: torch.Tensor, y: torch.Tensor, blank_id=0) -> l
         Returns:
             torch.Tensor: alignment result
         """
+        torchaudio = require_torchaudio("forced alignment")
         ctc_probs = ctc_probs[None].cpu()
         y = y[None].cpu()
         alignments, _ = torchaudio.functional.forced_align(ctc_probs, y, blank=blank_id)

funasr/utils/fbank.py

@@ -0,0 +1,166 @@
+# Copyright (c) community contributors.
+# A minimal fbank backend that lets FunASR run without torchaudio.
+#
+# torchaudio has no wheel matching the torch build on Ascend NPU (aarch64)
+# servers, where torch is a Huawei-forked nightly (e.g. 2.14.0a0) whose version
+# number is out of step with upstream PyTorch releases. This module forwards to
+# torchaudio.compliance.kaldi.fbank when torchaudio is present and otherwise
+# falls back to kaldi-native-fbank -- a standalone C++ Kaldi fbank
+# implementation with no torch dependency that ships manylinux aarch64 wheels.
+#
+# torchaudio.compliance.kaldi.fbank is only used on the CPU for feature
+# extraction. The fallback mirrors its signature so call sites only need to
+# change a single import line:
+#
+#     from funasr.utils import fbank as kaldi
+#
+# Not a full drop-in: the fallback supports single-channel waveforms and the
+# FbankOptions exposed by kaldi-native-fbank. torchaudio-only options such as
+# ``subtract_mean``, ``min_duration``, VTLN warping or an explicit
+# ``blackman_coeff`` are rejected with a clear error rather than silently
+# ignored, and multi-channel waveforms without an explicit ``channel`` are
+# rejected (kaldi-native-fbank is single-channel).
+import torch
+import numpy as np
+
+try:
+    import torchaudio.compliance.kaldi as _torchaudio_kaldi
+
+    _HAS_TORCHAUDIO = True
+except Exception:  # noqa: BLE001 -- torchaudio is optional here
+    _HAS_TORCHAUDIO = False
+
+_knf = None
+if not _HAS_TORCHAUDIO:
+    try:
+        import kaldi_native_fbank as _knf
+    except Exception:  # noqa: BLE001 -- reported lazily on first use
+        _knf = None
+
+
+def _knf_missing_error():
+    return ImportError(
+        "torchaudio is not installed and neither is the kaldi-native-fbank "
+        "fallback backend. FunASR needs one fbank backend for feature "
+        "extraction. Install either torchaudio (matching your torch version) "
+        "or kaldi-native-fbank, e.g. `pip install kaldi-native-fbank` (or "
+        "`pip install funasr[knf]`)."
+    )
+
+
+def _fbank_knf(
+    waveform: torch.Tensor,
+    blackman_coeff: float = 0.42,
+    channel: int = -1,
+    dither: float = 0.0,
+    energy_floor: float = 1.0,
+    frame_length: float = 25.0,
+    frame_shift: float = 10.0,
+    high_freq: float = 0.0,
+    htk_compat: bool = False,
+    low_freq: float = 20.0,
+    min_duration: float = 0.0,
+    num_mel_bins: int = 23,
+    preemphasis_coefficient: float = 0.97,
+    raw_energy: bool = True,
+    remove_dc_offset: bool = True,
+    round_to_power_of_two: bool = True,
+    sample_frequency: float = 16000.0,
+    snip_edges: bool = True,
+    subtract_mean: bool = False,
+    use_energy: bool = False,
+    use_log_fbank: bool = True,
+    use_power: bool = True,
+    vtln_high: float = -500.0,
+    vtln_low: float = 100.0,
+    vtln_warp: float = 1.0,
+    window_type: str = "povey",
+) -> torch.Tensor:
+    """kaldi-native-fbank implementation of ``torchaudio.compliance.kaldi.fbank``."""
+    if _knf is None:
+        raise _knf_missing_error()
+
+    # torchaudio options the kaldi-native-fbank fallback does not implement.
+    # Reject them explicitly instead of silently dropping them.
+    unsupported = {}
+    if blackman_coeff != 0.42:
+        unsupported["blackman_coeff"] = blackman_coeff
+    if min_duration != 0.0:
+        unsupported["min_duration"] = min_duration
+    if subtract_mean:
+        unsupported["subtract_mean"] = subtract_mean
+    if vtln_warp != 1.0:
+        unsupported["vtln_warp"] = vtln_warp
+    if vtln_low != 100.0:
+        unsupported["vtln_low"] = vtln_low
+    if vtln_high != -500.0:
+        unsupported["vtln_high"] = vtln_high
+    if unsupported:
+        raise NotImplementedError(
+            "The kaldi-native-fbank fallback does not support these "
+            f"torchaudio fbank options: {sorted(unsupported)}. Install "
+            "torchaudio to use them, or drop these options."
+        )
+
+    # Channel / waveform-shape handling, mirroring torchaudio semantics for the
+    # single-channel case kaldi-native-fbank supports.
+    if waveform.dim() == 2:
+        if channel >= 0:
+            waveform = waveform[channel]
+        elif waveform.size(0) == 1:
+            waveform = waveform[0]
+        else:
+            raise NotImplementedError(
+                "The kaldi-native-fbank fallback extracts features from a "
+                "single channel only. Pass `channel` to select one channel, or "
+                "reduce the waveform to mono before calling fbank()."
+            )
+    elif waveform.dim() == 1:
+        if channel > 0:
+            raise ValueError(
+                f"Invalid channel {channel} for a 1-D waveform; use channel=-1 or 0."
+            )
+    else:
+        raise ValueError("waveform must be a 1-D or 2-D tensor")
+
+    opts = _knf.FbankOptions()
+    opts.frame_opts.samp_freq = float(sample_frequency)
+    opts.frame_opts.frame_length_ms = float(frame_length)
+    opts.frame_opts.frame_shift_ms = float(frame_shift)
+    opts.frame_opts.dither = float(dither)
+    opts.frame_opts.snip_edges = bool(snip_edges)
+    opts.frame_opts.window_type = str(window_type)
+    opts.frame_opts.preemph_coeff = float(preemphasis_coefficient)
+    opts.frame_opts.remove_dc_offset = bool(remove_dc_offset)
+    opts.frame_opts.round_to_power_of_two = bool(round_to_power_of_two)
+    opts.mel_opts.num_bins = int(num_mel_bins)
+    opts.mel_opts.low_freq = float(low_freq)
+    opts.mel_opts.high_freq = float(high_freq)
+    opts.energy_floor = float(energy_floor)
+    opts.raw_energy = bool(raw_energy)
+    opts.use_energy = bool(use_energy)
+    opts.use_log_fbank = bool(use_log_fbank)
+    opts.use_power = bool(use_power)
+    opts.htk_compat = bool(htk_compat)
+
+    fb = _knf.OnlineFbank(opts)
+    wav = waveform.detach().cpu().numpy().reshape(-1).astype(np.float32)
+    fb.accept_waveform(int(sample_frequency), wav)
+    fb.input_finished()
+    n = fb.num_frames_ready
+    if n == 0:
+        return torch.zeros((0, int(num_mel_bins)), dtype=torch.float32)
+    frames = np.stack([fb.get_frame(i) for i in range(n)])
+    return torch.from_numpy(frames)
+
+
+def fbank(waveform: torch.Tensor, **kwargs) -> torch.Tensor:
+    """Drop-in replacement for :func:`torchaudio.compliance.kaldi.fbank`.
+
+    Forwards to torchaudio when it is installed; otherwise falls back to
+    kaldi-native-fbank, rejecting unsupported options explicitly (see module
+    docstring).
+    """
+    if _HAS_TORCHAUDIO:
+        return _torchaudio_kaldi.fbank(waveform, **kwargs)
+    return _fbank_knf(waveform, **kwargs)

funasr/utils/load_utils.py

@@ -6,7 +6,10 @@
 import numpy as np
 import kaldiio
 import librosa
-import torchaudio
+try:
+    import torchaudio
+except ImportError:
+    torchaudio = None
 import time
 import logging
 from torch.nn.utils.rnn import pad_sequence
@@ -174,8 +177,14 @@ def load_audio_text_image_video(
         # print(f"unsupport data type: {data_or_path_or_list}, return raw data")
 
     if audio_fs != fs and data_type != "text":
-        resampler = torchaudio.transforms.Resample(audio_fs, fs)
-        data_or_path_or_list = resampler(data_or_path_or_list[None, :])[0, :]
+        if torchaudio is not None:
+            resampler = torchaudio.transforms.Resample(audio_fs, fs)
+            data_or_path_or_list = resampler(data_or_path_or_list[None, :])[0, :]
+        else:
+            y = librosa.resample(
+                data_or_path_or_list.cpu().numpy(), orig_sr=int(audio_fs), target_sr=int(fs)
+            )
+            data_or_path_or_list = torch.from_numpy(y)
     return data_or_path_or_list
 
 

funasr/utils/speaker_utils.py

@@ -9,7 +9,7 @@
 import numpy as np
 import torch
 import torch.nn.functional as F
-import torchaudio.compliance.kaldi as Kaldi
+from funasr.utils import fbank as Kaldi
 from torch import nn
 
 from funasr.utils.modelscope_file import File

funasr/utils/torchaudio_compat.py

@@ -0,0 +1,38 @@
+# Copyright (c) community contributors.
+# Shared guard for the optional ``torchaudio`` dependency.
+#
+# FunASR's inference path no longer hard-depends on torchaudio: fbank feature
+# extraction falls back to kaldi-native-fbank (see :mod:`funasr.utils.fbank`)
+# and audio decoding falls back to soundfile/librosa. A few operations still
+# require torchaudio; call :func:`require_torchaudio` at their entry point so a
+# missing torchaudio fails with an actionable error instead of a bare
+# ``AttributeError`` (or, worse, a silently wrong result).
+import importlib.util
+
+_INSTALL_HINT = (
+    "Install a torchaudio build matching your torch version, e.g. "
+    "``pip install torchaudio``. Some platforms -- notably Ascend NPU "
+    "(aarch64) servers -- ship no matching torchaudio wheel; there, prefer a "
+    "FunASR path that does not need torchaudio."
+)
+
+
+def torchaudio_available() -> bool:
+    """Return ``True`` if torchaudio is importable."""
+    return importlib.util.find_spec("torchaudio") is not None
+
+
+def require_torchaudio(feature: str = "This operation"):
+    """Return the torchaudio module, or raise an actionable ``ImportError``.
+
+    Args:
+        feature: Human-readable name of the operation that needs torchaudio,
+            used in the error message.
+    """
+    try:
+        import torchaudio
+    except ImportError as exc:
+        raise ImportError(
+            f"{feature} requires torchaudio, which is not installed. {_INSTALL_HINT}"
+        ) from exc
+    return torchaudio