|
|
@ -20,11 +20,15 @@ from numpy import float32
|
|
|
|
from yacs.config import CfgNode
|
|
|
|
from yacs.config import CfgNode
|
|
|
|
|
|
|
|
|
|
|
|
from paddlespeech.cli.asr.infer import ASRExecutor
|
|
|
|
from paddlespeech.cli.asr.infer import ASRExecutor
|
|
|
|
|
|
|
|
from paddlespeech.cli.asr.infer import model_alias
|
|
|
|
|
|
|
|
from paddlespeech.cli.asr.infer import pretrained_models
|
|
|
|
from paddlespeech.cli.log import logger
|
|
|
|
from paddlespeech.cli.log import logger
|
|
|
|
from paddlespeech.cli.utils import MODEL_HOME
|
|
|
|
from paddlespeech.cli.utils import MODEL_HOME
|
|
|
|
from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
|
|
|
|
from paddlespeech.s2t.frontend.featurizer.text_featurizer import TextFeaturizer
|
|
|
|
from paddlespeech.s2t.frontend.speech import SpeechSegment
|
|
|
|
from paddlespeech.s2t.frontend.speech import SpeechSegment
|
|
|
|
from paddlespeech.s2t.modules.ctc import CTCDecoder
|
|
|
|
from paddlespeech.s2t.modules.ctc import CTCDecoder
|
|
|
|
|
|
|
|
from paddlespeech.s2t.transform.transformation import Transformation
|
|
|
|
|
|
|
|
from paddlespeech.s2t.utils.dynamic_import import dynamic_import
|
|
|
|
from paddlespeech.s2t.utils.utility import UpdateConfig
|
|
|
|
from paddlespeech.s2t.utils.utility import UpdateConfig
|
|
|
|
from paddlespeech.server.engine.base_engine import BaseEngine
|
|
|
|
from paddlespeech.server.engine.base_engine import BaseEngine
|
|
|
|
from paddlespeech.server.utils.audio_process import pcm2float
|
|
|
|
from paddlespeech.server.utils.audio_process import pcm2float
|
|
|
@ -51,6 +55,24 @@ pretrained_models = {
|
|
|
|
'lm_md5':
|
|
|
|
'lm_md5':
|
|
|
|
'29e02312deb2e59b3c8686c7966d4fe3'
|
|
|
|
'29e02312deb2e59b3c8686c7966d4fe3'
|
|
|
|
},
|
|
|
|
},
|
|
|
|
|
|
|
|
"conformer2online_aishell-zh-16k": {
|
|
|
|
|
|
|
|
'url':
|
|
|
|
|
|
|
|
'https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr1_chunk_conformer_aishell_ckpt_0.1.2.model.tar.gz',
|
|
|
|
|
|
|
|
'md5':
|
|
|
|
|
|
|
|
'4814e52e0fc2fd48899373f95c84b0c9',
|
|
|
|
|
|
|
|
'cfg_path':
|
|
|
|
|
|
|
|
'exp/chunk_conformer//conf/config.yaml',
|
|
|
|
|
|
|
|
'ckpt_path':
|
|
|
|
|
|
|
|
'exp/chunk_conformer/checkpoints/avg_30/',
|
|
|
|
|
|
|
|
'model':
|
|
|
|
|
|
|
|
'exp/chunk_conformer/checkpoints/avg_30.pdparams',
|
|
|
|
|
|
|
|
'params':
|
|
|
|
|
|
|
|
'exp/chunk_conformer/checkpoints/avg_30.pdparams',
|
|
|
|
|
|
|
|
'lm_url':
|
|
|
|
|
|
|
|
'https://deepspeech.bj.bcebos.com/zh_lm/zh_giga.no_cna_cmn.prune01244.klm',
|
|
|
|
|
|
|
|
'lm_md5':
|
|
|
|
|
|
|
|
'29e02312deb2e59b3c8686c7966d4fe3'
|
|
|
|
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -71,15 +93,17 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
Init model and other resources from a specific path.
|
|
|
|
Init model and other resources from a specific path.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
self.model_type = model_type
|
|
|
|
|
|
|
|
self.sample_rate = sample_rate
|
|
|
|
if cfg_path is None or am_model is None or am_params is None:
|
|
|
|
if cfg_path is None or am_model is None or am_params is None:
|
|
|
|
sample_rate_str = '16k' if sample_rate == 16000 else '8k'
|
|
|
|
sample_rate_str = '16k' if sample_rate == 16000 else '8k'
|
|
|
|
tag = model_type + '-' + lang + '-' + sample_rate_str
|
|
|
|
tag = model_type + '-' + lang + '-' + sample_rate_str
|
|
|
|
logger.info(f"Load the pretrained model, tag = {tag}")
|
|
|
|
logger.info(f"Load the pretrained model, tag = {tag}")
|
|
|
|
res_path = self._get_pretrained_path(tag) # wenetspeech_zh
|
|
|
|
res_path = self._get_pretrained_path(tag) # wenetspeech_zh
|
|
|
|
self.res_path = res_path
|
|
|
|
self.res_path = res_path
|
|
|
|
self.cfg_path = os.path.join(res_path,
|
|
|
|
self.cfg_path = "/home/users/xiongxinlei/task/paddlespeech-develop/PaddleSpeech/paddlespeech/server/tests/asr/online/conf/config.yaml"
|
|
|
|
pretrained_models[tag]['cfg_path'])
|
|
|
|
# self.cfg_path = os.path.join(res_path,
|
|
|
|
|
|
|
|
# pretrained_models[tag]['cfg_path'])
|
|
|
|
|
|
|
|
|
|
|
|
self.am_model = os.path.join(res_path,
|
|
|
|
self.am_model = os.path.join(res_path,
|
|
|
|
pretrained_models[tag]['model'])
|
|
|
|
pretrained_models[tag]['model'])
|
|
|
@ -119,13 +143,22 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
lm_url,
|
|
|
|
lm_url,
|
|
|
|
os.path.dirname(self.config.decode.lang_model_path), lm_md5)
|
|
|
|
os.path.dirname(self.config.decode.lang_model_path), lm_md5)
|
|
|
|
elif "conformer" in model_type or "transformer" in model_type or "wenetspeech" in model_type:
|
|
|
|
elif "conformer" in model_type or "transformer" in model_type or "wenetspeech" in model_type:
|
|
|
|
# 开发 conformer 的流式模型
|
|
|
|
|
|
|
|
logger.info("start to create the stream conformer asr engine")
|
|
|
|
logger.info("start to create the stream conformer asr engine")
|
|
|
|
# 复用cli里面的代码
|
|
|
|
if self.config.spm_model_prefix:
|
|
|
|
|
|
|
|
self.config.spm_model_prefix = os.path.join(
|
|
|
|
|
|
|
|
self.res_path, self.config.spm_model_prefix)
|
|
|
|
|
|
|
|
self.config.vocab_filepath = os.path.join(
|
|
|
|
|
|
|
|
self.res_path, self.config.vocab_filepath)
|
|
|
|
|
|
|
|
self.text_feature = TextFeaturizer(
|
|
|
|
|
|
|
|
unit_type=self.config.unit_type,
|
|
|
|
|
|
|
|
vocab=self.config.vocab_filepath,
|
|
|
|
|
|
|
|
spm_model_prefix=self.config.spm_model_prefix)
|
|
|
|
|
|
|
|
# update the decoding method
|
|
|
|
|
|
|
|
if decode_method:
|
|
|
|
|
|
|
|
self.config.decode.decoding_method = decode_method
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise Exception("wrong type")
|
|
|
|
raise Exception("wrong type")
|
|
|
|
|
|
|
|
if "deepspeech2online" in model_type or "deepspeech2offline" in model_type:
|
|
|
|
# AM predictor
|
|
|
|
# AM predictor
|
|
|
|
logger.info("ASR engine start to init the am predictor")
|
|
|
|
logger.info("ASR engine start to init the am predictor")
|
|
|
|
self.am_predictor_conf = am_predictor_conf
|
|
|
|
self.am_predictor_conf = am_predictor_conf
|
|
|
@ -162,6 +195,15 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
self.chunk_state_c_box = np.zeros(
|
|
|
|
self.chunk_state_c_box = np.zeros(
|
|
|
|
(self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
|
|
|
|
(self.config.num_rnn_layers, 1, self.config.rnn_layer_size),
|
|
|
|
dtype=float32)
|
|
|
|
dtype=float32)
|
|
|
|
|
|
|
|
elif "conformer" in model_type or "transformer" in model_type or "wenetspeech" in model_type:
|
|
|
|
|
|
|
|
model_name = model_type[:model_type.rindex(
|
|
|
|
|
|
|
|
'_')] # model_type: {model_name}_{dataset}
|
|
|
|
|
|
|
|
logger.info(f"model name: {model_name}")
|
|
|
|
|
|
|
|
model_class = dynamic_import(model_name, model_alias)
|
|
|
|
|
|
|
|
model_conf = self.config
|
|
|
|
|
|
|
|
model = model_class.from_config(model_conf)
|
|
|
|
|
|
|
|
self.model = model
|
|
|
|
|
|
|
|
logger.info("create the transformer like model success")
|
|
|
|
|
|
|
|
|
|
|
|
def reset_decoder_and_chunk(self):
|
|
|
|
def reset_decoder_and_chunk(self):
|
|
|
|
"""reset decoder and chunk state for an new audio
|
|
|
|
"""reset decoder and chunk state for an new audio
|
|
|
@ -186,6 +228,7 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
Returns:
|
|
|
|
Returns:
|
|
|
|
[type]: [description]
|
|
|
|
[type]: [description]
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
logger.info("start to decoce chunk by chunk")
|
|
|
|
if "deepspeech2online" in model_type:
|
|
|
|
if "deepspeech2online" in model_type:
|
|
|
|
input_names = self.am_predictor.get_input_names()
|
|
|
|
input_names = self.am_predictor.get_input_names()
|
|
|
|
audio_handle = self.am_predictor.get_input_handle(input_names[0])
|
|
|
|
audio_handle = self.am_predictor.get_input_handle(input_names[0])
|
|
|
@ -224,10 +267,29 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
|
|
|
|
|
|
|
|
self.decoder.next(output_chunk_probs, output_chunk_lens)
|
|
|
|
self.decoder.next(output_chunk_probs, output_chunk_lens)
|
|
|
|
trans_best, trans_beam = self.decoder.decode()
|
|
|
|
trans_best, trans_beam = self.decoder.decode()
|
|
|
|
|
|
|
|
logger.info(f"decode one one best result: {trans_best[0]}")
|
|
|
|
return trans_best[0]
|
|
|
|
return trans_best[0]
|
|
|
|
|
|
|
|
|
|
|
|
elif "conformer" in model_type or "transformer" in model_type:
|
|
|
|
elif "conformer" in model_type or "transformer" in model_type:
|
|
|
|
raise Exception("invalid model name")
|
|
|
|
try:
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
|
|
|
f"we will use the transformer like model : {self.model_type}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
cfg = self.config.decode
|
|
|
|
|
|
|
|
result_transcripts = self.model.decode(
|
|
|
|
|
|
|
|
x_chunk,
|
|
|
|
|
|
|
|
x_chunk_lens,
|
|
|
|
|
|
|
|
text_feature=self.text_feature,
|
|
|
|
|
|
|
|
decoding_method=cfg.decoding_method,
|
|
|
|
|
|
|
|
beam_size=cfg.beam_size,
|
|
|
|
|
|
|
|
ctc_weight=cfg.ctc_weight,
|
|
|
|
|
|
|
|
decoding_chunk_size=cfg.decoding_chunk_size,
|
|
|
|
|
|
|
|
num_decoding_left_chunks=cfg.num_decoding_left_chunks,
|
|
|
|
|
|
|
|
simulate_streaming=cfg.simulate_streaming)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return result_transcripts[0][0]
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
logger.exception(e)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
raise Exception("invalid model name")
|
|
|
|
raise Exception("invalid model name")
|
|
|
|
|
|
|
|
|
|
|
@ -244,7 +306,7 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
# pcm16 -> pcm 32
|
|
|
|
# pcm16 -> pcm 32
|
|
|
|
samples = pcm2float(samples)
|
|
|
|
samples = pcm2float(samples)
|
|
|
|
|
|
|
|
if "deepspeech2online" in self.model_type:
|
|
|
|
# read audio
|
|
|
|
# read audio
|
|
|
|
speech_segment = SpeechSegment.from_pcm(
|
|
|
|
speech_segment = SpeechSegment.from_pcm(
|
|
|
|
samples, sample_rate, transcript=" ")
|
|
|
|
samples, sample_rate, transcript=" ")
|
|
|
@ -259,7 +321,8 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
spectrum = self.collate_fn_test._normalizer.apply(spectrum)
|
|
|
|
spectrum = self.collate_fn_test._normalizer.apply(spectrum)
|
|
|
|
|
|
|
|
|
|
|
|
# spectrum augment
|
|
|
|
# spectrum augment
|
|
|
|
audio = self.collate_fn_test.augmentation.transform_feature(spectrum)
|
|
|
|
audio = self.collate_fn_test.augmentation.transform_feature(
|
|
|
|
|
|
|
|
spectrum)
|
|
|
|
|
|
|
|
|
|
|
|
audio_len = audio.shape[0]
|
|
|
|
audio_len = audio.shape[0]
|
|
|
|
audio = paddle.to_tensor(audio, dtype='float32')
|
|
|
|
audio = paddle.to_tensor(audio, dtype='float32')
|
|
|
@ -270,6 +333,28 @@ class ASRServerExecutor(ASRExecutor):
|
|
|
|
x_chunk_lens = np.array([audio_len])
|
|
|
|
x_chunk_lens = np.array([audio_len])
|
|
|
|
|
|
|
|
|
|
|
|
return x_chunk, x_chunk_lens
|
|
|
|
return x_chunk, x_chunk_lens
|
|
|
|
|
|
|
|
elif "conformer2online" in self.model_type:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if sample_rate != self.sample_rate:
|
|
|
|
|
|
|
|
logger.info(f"audio sample rate {sample_rate} is not match," \
|
|
|
|
|
|
|
|
"the model sample_rate is {self.sample_rate}")
|
|
|
|
|
|
|
|
logger.info(f"ASR Engine use the {self.model_type} to process")
|
|
|
|
|
|
|
|
logger.info("Create the preprocess instance")
|
|
|
|
|
|
|
|
preprocess_conf = self.config.preprocess_config
|
|
|
|
|
|
|
|
preprocess_args = {"train": False}
|
|
|
|
|
|
|
|
preprocessing = Transformation(preprocess_conf)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("Read the audio file")
|
|
|
|
|
|
|
|
logger.info(f"audio shape: {samples.shape}")
|
|
|
|
|
|
|
|
# fbank
|
|
|
|
|
|
|
|
x_chunk = preprocessing(samples, **preprocess_args)
|
|
|
|
|
|
|
|
x_chunk_lens = paddle.to_tensor(x_chunk.shape[0])
|
|
|
|
|
|
|
|
x_chunk = paddle.to_tensor(
|
|
|
|
|
|
|
|
x_chunk, dtype="float32").unsqueeze(axis=0)
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
|
|
|
f"process the audio feature success, feat shape: {x_chunk.shape}"
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
return x_chunk, x_chunk_lens
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ASREngine(BaseEngine):
|
|
|
|
class ASREngine(BaseEngine):
|
|
|
@ -310,7 +395,10 @@ class ASREngine(BaseEngine):
|
|
|
|
logger.info("Initialize ASR server engine successfully.")
|
|
|
|
logger.info("Initialize ASR server engine successfully.")
|
|
|
|
return True
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
|
|
def preprocess(self, samples, sample_rate):
|
|
|
|
def preprocess(self,
|
|
|
|
|
|
|
|
samples,
|
|
|
|
|
|
|
|
sample_rate,
|
|
|
|
|
|
|
|
model_type="deepspeech2online_aishell-zh-16k"):
|
|
|
|
"""preprocess
|
|
|
|
"""preprocess
|
|
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
Args:
|
|
|
@ -321,6 +409,7 @@ class ASREngine(BaseEngine):
|
|
|
|
x_chunk (numpy.array): shape[B, T, D]
|
|
|
|
x_chunk (numpy.array): shape[B, T, D]
|
|
|
|
x_chunk_lens (numpy.array): shape[B]
|
|
|
|
x_chunk_lens (numpy.array): shape[B]
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
# if "deepspeech" in model_type:
|
|
|
|
x_chunk, x_chunk_lens = self.executor.extract_feat(samples, sample_rate)
|
|
|
|
x_chunk, x_chunk_lens = self.executor.extract_feat(samples, sample_rate)
|
|
|
|
return x_chunk, x_chunk_lens
|
|
|
|
return x_chunk, x_chunk_lens
|
|
|
|
|
|
|
|
|
|
|
|