Add support for fun-asr model to accuracy_checker (#4063)

* Prepared draft of HfFunAsrPipeline

* Added fix funasr for input format conversion.

* Updated ASROptimumPipeline to add support for fun-asr model.

* Added FunAsrOptimumPipeline

* Applied changes to finetune fun asr evaluator pipeline

* Removed data normalization in ARS pipeline and return_timestamps

* Removed redundant code

* Updated language handling in GenAIASRPipeline

* Updated readme

* Updated ASR classes naming

* Updated readme

* Updated imports in asr_pipeline_evaluator

* Added test_hf_fun_asr_pipeline

---------

Co-authored-by: Pawel Kowalczyk <pawel.kowalczyk>

Author: pkowalczint

tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/README.md

@@ -65,5 +65,6 @@ Optionally you can provide `module_config` section which contains config for cus
 * **Whisper Evaluator** demonstrates how to evaluate Whisper family models with various pipeline classes, including GenAIWhisperPipeline, HFWhisperPipeline, and OptimumWhisperPipeline.
   Evaluator code: `<omz_dir>/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py`.
 
-* **ASR Pipeline Evaluator** demonstrates how to evaluate automatic speech recognition models with various pipeline classes, including GenAIASRPipeline, Qwen3ASROptimumPipeline
+* **ASR Pipeline Evaluator** demonstrates how to evaluate automatic speech recognition models with various ASR pipeline classes, including GenAIASRPipeline,
+Qwen3ASROptimumPipeline and FunASROptimumPipeline
   Evaluator code: `<omz_dir>/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/asr_pipeline_evaluator.py`.

tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/asr_pipeline_evaluator.py

@@ -17,6 +17,8 @@
 import importlib
 import re
 
+import numpy as np
+
 from ...representation import CharacterRecognitionPrediction
 from ...utils import UnsupportedPackage, extract_image_representations
 from .base_custom_evaluator import BaseCustomEvaluator
@@ -32,7 +34,9 @@ class ASRPipelineEvaluator(BaseCustomEvaluator):
     VALID_PIPELINE_CLASSES = [
         "GenAIASRPipeline",
         "HFASRPipeline",
+        "HFFunASRPipeline",
         "Qwen3ASROptimumPipeline",
+        "FunASROptimumPipeline",
     ]
 
     def __init__(self, dataset_config, pipe, orig_config):
@@ -123,6 +127,10 @@ def predict(self, identifiers, input_data, input_meta, encoder_callback=None):
 
 
 class GenAIASRPipeline(ASRPipeline):
+    def __init__(self, config):
+        self.language = config.get("language")
+        super().__init__(config)
+
     def _initialize_pipeline(self, config):
         try:
             import openvino_genai as ov_genai  # pylint: disable=C0415
@@ -136,9 +144,11 @@ def _initialize_pipeline(self, config):
         return ov_genai.ASRPipeline(str(model_dir), device=device)
 
     def _get_predictions(self, data, identifier, input_meta):
+        kwargs = {"language": self.language} if self.language else {}
         return self.pipeline.generate(
             data[0],
             return_timestamps=True,
+            **kwargs,
         ).texts[0]
 
 
@@ -228,6 +238,60 @@ def parse_asr_output(raw_text):
         }
 
 
+class FunASROptimumPipeline(ASRPipeline):
+    SAMPLE_RATE = 16000
+    EOS_TOKEN_IDS = [151643, 151645]
+
+    def __init__(self, config):
+        self.max_new_tokens = config.get("max_new_tokens", 1000)
+        # FunASR prompt language name, e.g. "英文" (English), "中文" (Chinese)
+        self.language = config.get("language", "英文")
+        self.itn = config.get("itn", True)
+        super().__init__(config)
+
+    def _initialize_pipeline(self, config):
+        try:
+            from optimum.intel.openvino import OVModelForSpeechSeq2Seq  # pylint: disable=C0415
+        except ImportError as import_error:
+            UnsupportedPackage("optimum.intel.openvino", import_error.msg).raise_error(
+                self.__class__.__name__
+            )
+
+        try:
+            from transformers import AutoTokenizer  # pylint: disable=C0415
+        except ImportError as import_error:
+            UnsupportedPackage("transformers", import_error.msg).raise_error(
+                self.__class__.__name__
+            )
+
+        model_dir = get_model_dir(config)
+        device = config.get("_device", "CPU")
+        ov_model = OVModelForSpeechSeq2Seq.from_pretrained(str(model_dir)).to(device)
+        tokenizer = AutoTokenizer.from_pretrained(str(model_dir))
+        return ov_model, tokenizer
+
+    def _get_predictions(self, data, identifier, input_meta):
+        ov_model, tokenizer = self.pipeline
+        sampling_rate = input_meta[0].get("sample_rate", self.SAMPLE_RATE)
+        audio = np.ascontiguousarray(data[0], dtype=np.float32).flatten()
+        inputs = ov_model.preprocess_input(
+            audio, sampling_rate=sampling_rate, language=self.language, itn=self.itn
+        )
+
+        output_ids = ov_model.generate(
+            input_features=inputs["input_features"],
+            decoder_input_ids=inputs["decoder_input_ids"],
+            decoder_attention_mask=inputs["decoder_attention_mask"],
+            eos_token_id=self.EOS_TOKEN_IDS,
+            max_new_tokens=self.max_new_tokens,
+        )
+
+        prompt_len = inputs["decoder_input_ids"].shape[1]
+        return tokenizer.batch_decode(
+            output_ids[:, prompt_len:], skip_special_tokens=True
+        )[0].strip()
+
+
 class HFASRPipeline(ASRPipeline):
     def __init__(self, config):
         self.language = config.get("language")
@@ -305,3 +369,42 @@ def _get_predictions(self, data, identifier, input_meta):
                 "num_beams": 1,
             },
         )["text"]
+
+class HFFunASRPipeline(ASRPipeline):
+    def __init__(self, config):
+        self.device = config.get("_device", "cpu").lower()
+        # FunASR prompt language name, e.g. "英文" (English), "中文" (Chinese)
+        self.language = config.get("language", "英文")
+        self.itn = config.get("itn", True)
+        super().__init__(config)
+
+    def _initialize_pipeline(self, config):
+        try:
+            from funasr import AutoModel  # pylint: disable=C0415
+        except ImportError as ie:
+            UnsupportedPackage("funasr", ie.msg).raise_error(self.__class__.__name__)
+
+        model_id = config.get("model_id")
+        model = AutoModel(
+            model=model_id,
+            hub="hf",
+            trust_remote_code=True,
+            device=self.device
+        )
+        return model
+
+    def _get_predictions(self, data, identifier, input_meta):
+        try:
+            import torch  # pylint: disable=C0415
+        except ImportError as ie:
+            UnsupportedPackage("torch", ie.msg).raise_error(self.__class__.__name__)
+
+        audio = torch.from_numpy(np.ascontiguousarray(data[0], dtype=np.float32)).flatten()
+        res = self.pipeline.generate(
+            input=[audio],
+            cache={},
+            hotwords=[],
+            language=self.language,
+            itn=self.itn,
+        )
+        return res[0]["text"]

tools/accuracy_checker/tests/test_asr_pipeline_evaluator.py

@@ -20,7 +20,8 @@
 from accuracy_checker.evaluators.custom_evaluators.asr_pipeline_evaluator import (
     ASRPipelineEvaluator,
     GenAIASRPipeline,
-    HFASRPipeline
+    HFASRPipeline,
+    HFFunASRPipeline,
 )
 from datasets import load_dataset
 
@@ -107,3 +108,11 @@ def test_hf_asr_pipeline_whisper_tiny(self):
 
         result = evaluator.pipe._get_predictions(input_data, identifiers[0], input_meta)
         assert isinstance(result, str)
+
+    def test_hf_fun_asr_pipeline(self):
+        config = {"_device": "CPU", "model_id": "FunAudioLLM/Fun-ASR-Nano-2512", "language": "英文"}
+        pipeline = HFFunASRPipeline(config)
+        evaluator = ASRPipelineEvaluator(None, pipeline, None)
+
+        result = evaluator.pipe._get_predictions(input_data, identifiers[0], input_meta)
+        assert isinstance(result, str)