Merge pull request #1906 from Honei/acs_server
[acs][server]add audio content search serverpull/1919/head
Honei
3 years ago
committed by
GitHub
commit
bde7093578
@ -0,0 +1,74 @@
|
||||
([简体中文](./README_cn.md)|English)
|
||||
# ACS (Audio Content Search)
|
||||
|
||||
## Introduction
|
||||
ACS, or Audio Content Search, refers to the problem of getting the key word time stamp from automatically transcribe spoken language (speech-to-text).
|
||||
|
||||
This demo is an implementation of obtaining the keyword timestamp in the text from a given audio file. It can be done by a single command or a few lines in python using `PaddleSpeech`.
|
||||
Now, the search word in demo is:
|
||||
```
|
||||
我
|
||||
康
|
||||
```
|
||||
## Usage
|
||||
### 1. Installation
|
||||
see [installation](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/docs/source/install.md).
|
||||
|
||||
You can choose one way from meduim and hard to install paddlespeech.
|
||||
|
||||
The dependency refers to the requirements.txt
|
||||
### 2. Prepare Input File
|
||||
The input of this demo should be a WAV file(`.wav`), and the sample rate must be the same as the model.
|
||||
|
||||
Here are sample files for this demo that can be downloaded:
|
||||
```bash
|
||||
wget -c https://paddlespeech.bj.bcebos.com/PaddleAudio/zh.wav
|
||||
```
|
||||
|
||||
### 3. Usage
|
||||
- Command Line(Recommended)
|
||||
```bash
|
||||
# Chinese
|
||||
paddlespeech_client acs --server_ip 127.0.0.1 --port 8090 --input ./zh.wav
|
||||
```
|
||||
|
||||
Usage:
|
||||
```bash
|
||||
paddlespeech asr --help
|
||||
```
|
||||
Arguments:
|
||||
- `input`(required): Audio file to recognize.
|
||||
- `server_ip`: the server ip.
|
||||
- `port`: the server port.
|
||||
- `lang`: the language type of the model. Default: `zh`.
|
||||
- `sample_rate`: Sample rate of the model. Default: `16000`.
|
||||
- `audio_format`: The audio format.
|
||||
|
||||
Output:
|
||||
```bash
|
||||
[2022-05-15 15:00:58,185] [ INFO] - acs http client start
|
||||
[2022-05-15 15:00:58,185] [ INFO] - endpoint: http://127.0.0.1:8490/paddlespeech/asr/search
|
||||
[2022-05-15 15:01:03,220] [ INFO] - acs http client finished
|
||||
[2022-05-15 15:01:03,221] [ INFO] - ACS result: {'transcription': '我认为跑步最重要的就是给我带来了身体健康', 'acs': [{'w': '我', 'bg': 0, 'ed': 1.6800000000000002}, {'w': '我', 'bg': 2.1, 'ed': 4.28}, {'w': '康', 'bg': 3.2, 'ed': 4.92}]}
|
||||
[2022-05-15 15:01:03,221] [ INFO] - Response time 5.036084 s.
|
||||
```
|
||||
|
||||
- Python API
|
||||
```python
|
||||
from paddlespeech.server.bin.paddlespeech_client import ACSClientExecutor
|
||||
|
||||
acs_executor = ACSClientExecutor()
|
||||
res = acs_executor(
|
||||
input='./zh.wav',
|
||||
server_ip="127.0.0.1",
|
||||
port=8490,)
|
||||
print(res)
|
||||
```
|
||||
|
||||
Output:
|
||||
```bash
|
||||
[2022-05-15 15:08:13,955] [ INFO] - acs http client start
|
||||
[2022-05-15 15:08:13,956] [ INFO] - endpoint: http://127.0.0.1:8490/paddlespeech/asr/search
|
||||
[2022-05-15 15:08:19,026] [ INFO] - acs http client finished
|
||||
{'transcription': '我认为跑步最重要的就是给我带来了身体健康', 'acs': [{'w': '我', 'bg': 0, 'ed': 1.6800000000000002}, {'w': '我', 'bg': 2.1, 'ed': 4.28}, {'w': '康', 'bg': 3.2, 'ed': 4.92}]}
|
||||
```
|
||||
@ -0,0 +1,49 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import argparse
|
||||
|
||||
from paddlespeech.cli.log import logger
|
||||
from paddlespeech.server.utils.audio_handler import ASRHttpHandler
|
||||
|
||||
|
||||
def main(args):
|
||||
logger.info("asr http client start")
|
||||
audio_format = "wav"
|
||||
sample_rate = 16000
|
||||
lang = "zh"
|
||||
handler = ASRHttpHandler(
|
||||
server_ip=args.server_ip, port=args.port, endpoint=args.endpoint)
|
||||
res = handler.run(args.wavfile, audio_format, sample_rate, lang)
|
||||
# res = res['result']
|
||||
logger.info(f"the final result: {res}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="audio content search client")
|
||||
parser.add_argument(
|
||||
'--server_ip', type=str, default='127.0.0.1', help='server ip')
|
||||
parser.add_argument('--port', type=int, default=8090, help='server port')
|
||||
parser.add_argument(
|
||||
"--wavfile",
|
||||
action="store",
|
||||
help="wav file path ",
|
||||
default="./16_audio.wav")
|
||||
parser.add_argument(
|
||||
'--endpoint',
|
||||
type=str,
|
||||
default='/paddlespeech/asr/search',
|
||||
help='server endpoint')
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
@ -0,0 +1,34 @@
|
||||
#################################################################################
|
||||
# SERVER SETTING #
|
||||
#################################################################################
|
||||
host: 0.0.0.0
|
||||
port: 8490
|
||||
|
||||
# The task format in the engin_list is: <speech task>_<engine type>
|
||||
# task choices = ['acs_python']
|
||||
# protocol = ['http'] (only one can be selected).
|
||||
# http only support offline engine type.
|
||||
protocol: 'http'
|
||||
engine_list: ['acs_python']
|
||||
|
||||
|
||||
#################################################################################
|
||||
# ENGINE CONFIG #
|
||||
#################################################################################
|
||||
|
||||
################################### ACS #########################################
|
||||
################### acs task: engine_type: python ###############################
|
||||
acs_python:
|
||||
task: acs
|
||||
asr_protocol: 'websocket' # 'websocket'
|
||||
offset: 1.0 # second
|
||||
asr_server_ip: 127.0.0.1
|
||||
asr_server_port: 8390
|
||||
lang: 'zh'
|
||||
word_list: "./conf/words.txt"
|
||||
sample_rate: 16000
|
||||
device: 'cpu' # set 'gpu:id' or 'cpu'
|
||||
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
我
|
||||
康
|
||||
@ -0,0 +1,43 @@
|
||||
#################################################################################
|
||||
# SERVER SETTING #
|
||||
#################################################################################
|
||||
host: 0.0.0.0
|
||||
port: 8390
|
||||
|
||||
# The task format in the engin_list is: <speech task>_<engine type>
|
||||
# task choices = ['asr_online']
|
||||
# protocol = ['websocket'] (only one can be selected).
|
||||
# websocket only support online engine type.
|
||||
protocol: 'websocket'
|
||||
engine_list: ['asr_online']
|
||||
|
||||
|
||||
#################################################################################
|
||||
# ENGINE CONFIG #
|
||||
#################################################################################
|
||||
|
||||
################################### ASR #########################################
|
||||
################### speech task: asr; engine_type: online #######################
|
||||
asr_online:
|
||||
model_type: 'conformer_online_multicn'
|
||||
am_model: # the pdmodel file of am static model [optional]
|
||||
am_params: # the pdiparams file of am static model [optional]
|
||||
lang: 'zh'
|
||||
sample_rate: 16000
|
||||
cfg_path:
|
||||
decode_method: 'attention_rescoring'
|
||||
force_yes: True
|
||||
device: 'cpu' # cpu or gpu:id
|
||||
am_predictor_conf:
|
||||
device: # set 'gpu:id' or 'cpu'
|
||||
switch_ir_optim: True
|
||||
glog_info: False # True -> print glog
|
||||
summary: True # False -> do not show predictor config
|
||||
|
||||
chunk_buffer_conf:
|
||||
window_n: 7 # frame
|
||||
shift_n: 4 # frame
|
||||
window_ms: 25 # ms
|
||||
shift_ms: 10 # ms
|
||||
sample_rate: 16000
|
||||
sample_width: 2
|
||||
@ -0,0 +1,46 @@
|
||||
# This is the parameter configuration file for PaddleSpeech Serving.
|
||||
|
||||
#################################################################################
|
||||
# SERVER SETTING #
|
||||
#################################################################################
|
||||
host: 0.0.0.0
|
||||
port: 8390
|
||||
|
||||
# The task format in the engin_list is: <speech task>_<engine type>
|
||||
# task choices = ['asr_online']
|
||||
# protocol = ['websocket'] (only one can be selected).
|
||||
# websocket only support online engine type.
|
||||
protocol: 'websocket'
|
||||
engine_list: ['asr_online']
|
||||
|
||||
|
||||
#################################################################################
|
||||
# ENGINE CONFIG #
|
||||
#################################################################################
|
||||
|
||||
################################### ASR #########################################
|
||||
################### speech task: asr; engine_type: online #######################
|
||||
asr_online:
|
||||
model_type: 'conformer_online_wenetspeech'
|
||||
am_model: # the pdmodel file of am static model [optional]
|
||||
am_params: # the pdiparams file of am static model [optional]
|
||||
lang: 'zh'
|
||||
sample_rate: 16000
|
||||
cfg_path:
|
||||
decode_method:
|
||||
force_yes: True
|
||||
device: 'cpu' # cpu or gpu:id
|
||||
decode_method: "attention_rescoring"
|
||||
am_predictor_conf:
|
||||
device: # set 'gpu:id' or 'cpu'
|
||||
switch_ir_optim: True
|
||||
glog_info: False # True -> print glog
|
||||
summary: True # False -> do not show predictor config
|
||||
|
||||
chunk_buffer_conf:
|
||||
window_n: 7 # frame
|
||||
shift_n: 4 # frame
|
||||
window_ms: 25 # ms
|
||||
shift_ms: 10 # ms
|
||||
sample_rate: 16000
|
||||
sample_width: 2
|
||||
@ -0,0 +1,7 @@
|
||||
export CUDA_VISIBLE_DEVICE=0,1,2,3
|
||||
# we need the streaming asr server
|
||||
nohup python3 streaming_asr_server.py --config_file conf/ws_conformer_application.yaml > streaming_asr.log 2>&1 &
|
||||
|
||||
# start the acs server
|
||||
nohup paddlespeech_server start --config_file conf/acs_application.yaml > acs.log 2>&1 &
|
||||
|
||||
@ -0,0 +1,188 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import paddle
|
||||
import soundfile
|
||||
import websocket
|
||||
|
||||
from paddlespeech.cli.log import logger
|
||||
from paddlespeech.server.engine.base_engine import BaseEngine
|
||||
|
||||
|
||||
class ACSEngine(BaseEngine):
|
||||
def __init__(self):
|
||||
"""The ACSEngine Engine
|
||||
"""
|
||||
super(ACSEngine, self).__init__()
|
||||
logger.info("Create the ACSEngine Instance")
|
||||
self.word_list = []
|
||||
|
||||
def init(self, config: dict):
|
||||
"""Init the ACSEngine Engine
|
||||
|
||||
Args:
|
||||
config (dict): The server configuation
|
||||
|
||||
Returns:
|
||||
bool: The engine instance flag
|
||||
"""
|
||||
logger.info("Init the acs engine")
|
||||
try:
|
||||
self.config = config
|
||||
if self.config.device:
|
||||
self.device = self.config.device
|
||||
else:
|
||||
self.device = paddle.get_device()
|
||||
|
||||
paddle.set_device(self.device)
|
||||
logger.info(f"ACS Engine set the device: {self.device}")
|
||||
|
||||
except BaseException as e:
|
||||
logger.error(
|
||||
"Set device failed, please check if device is already used and the parameter 'device' in the yaml file"
|
||||
)
|
||||
logger.error("Initialize Text server engine Failed on device: %s." %
|
||||
(self.device))
|
||||
return False
|
||||
|
||||
self.read_search_words()
|
||||
|
||||
# init the asr url
|
||||
self.url = "ws://" + self.config.asr_server_ip + ":" + str(
|
||||
self.config.asr_server_port) + "/paddlespeech/asr/streaming"
|
||||
|
||||
logger.info("Init the acs engine successfully")
|
||||
return True
|
||||
|
||||
def read_search_words(self):
|
||||
word_list = self.config.word_list
|
||||
if word_list is None:
|
||||
logger.error(
|
||||
"No word list file in config, please set the word list parameter"
|
||||
)
|
||||
return
|
||||
|
||||
if not os.path.exists(word_list):
|
||||
logger.error("Please input correct word list file")
|
||||
return
|
||||
|
||||
with open(word_list, 'r') as fp:
|
||||
self.word_list = [line.strip() for line in fp.readlines()]
|
||||
|
||||
logger.info(f"word list: {self.word_list}")
|
||||
|
||||
def get_asr_content(self, audio_data):
|
||||
"""Get the streaming asr result
|
||||
|
||||
Args:
|
||||
audio_data (_type_): _description_
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
logger.info("send a message to the server")
|
||||
if self.url is None:
|
||||
logger.error("No asr server, please input valid ip and port")
|
||||
return ""
|
||||
ws = websocket.WebSocket()
|
||||
ws.connect(self.url)
|
||||
# with websocket.WebSocket.connect(self.url) as ws:
|
||||
audio_info = json.dumps(
|
||||
{
|
||||
"name": "test.wav",
|
||||
"signal": "start",
|
||||
"nbest": 1
|
||||
},
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
separators=(',', ': '))
|
||||
ws.send(audio_info)
|
||||
msg = ws.recv()
|
||||
logger.info("client receive msg={}".format(msg))
|
||||
|
||||
# send the total audio data
|
||||
samples, sample_rate = soundfile.read(audio_data, dtype='int16')
|
||||
ws.send_binary(samples.tobytes())
|
||||
msg = ws.recv()
|
||||
msg = json.loads(msg)
|
||||
logger.info(f"audio result: {msg}")
|
||||
|
||||
# 3. send chunk audio data to engine
|
||||
logger.info("send the end signal")
|
||||
audio_info = json.dumps(
|
||||
{
|
||||
"name": "test.wav",
|
||||
"signal": "end",
|
||||
"nbest": 1
|
||||
},
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
separators=(',', ': '))
|
||||
ws.send(audio_info)
|
||||
msg = ws.recv()
|
||||
msg = json.loads(msg)
|
||||
|
||||
logger.info(f"the final result: {msg}")
|
||||
ws.close()
|
||||
|
||||
return msg
|
||||
|
||||
def get_macthed_word(self, msg):
|
||||
"""Get the matched info in msg
|
||||
|
||||
Args:
|
||||
msg (dict): the asr info, including the asr result and time stamp
|
||||
|
||||
Returns:
|
||||
acs_result, asr_result: the acs result and the asr result
|
||||
"""
|
||||
asr_result = msg['result']
|
||||
time_stamp = msg['times']
|
||||
acs_result = []
|
||||
|
||||
# search for each word in self.word_list
|
||||
offset = self.config.offset
|
||||
max_ed = time_stamp[-1]['ed']
|
||||
for w in self.word_list:
|
||||
# search the w in asr_result and the index in asr_result
|
||||
for m in re.finditer(w, asr_result):
|
||||
start = max(time_stamp[m.start(0)]['bg'] - offset, 0)
|
||||
|
||||
end = min(time_stamp[m.end(0) - 1]['ed'] + offset, max_ed)
|
||||
logger.info(f'start: {start}, end: {end}')
|
||||
acs_result.append({'w': w, 'bg': start, 'ed': end})
|
||||
|
||||
return acs_result, asr_result
|
||||
|
||||
def run(self, audio_data):
|
||||
"""process the audio data in acs engine
|
||||
the engine does not store any data, so all the request use the self.run api
|
||||
|
||||
Args:
|
||||
audio_data (str): the audio data
|
||||
|
||||
Returns:
|
||||
acs_result, asr_result: the acs result and the asr result
|
||||
"""
|
||||
logger.info("start to process the audio content search")
|
||||
msg = self.get_asr_content(io.BytesIO(audio_data))
|
||||
|
||||
acs_result, asr_result = self.get_macthed_word(msg)
|
||||
logger.info(f'the asr result {asr_result}')
|
||||
logger.info(f'the acs result: {acs_result}')
|
||||
return acs_result, asr_result
|
||||
@ -0,0 +1,101 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import base64
|
||||
from typing import Union
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from paddlespeech.cli.log import logger
|
||||
from paddlespeech.server.engine.engine_pool import get_engine_pool
|
||||
from paddlespeech.server.restful.request import ASRRequest
|
||||
from paddlespeech.server.restful.response import ACSResponse
|
||||
from paddlespeech.server.restful.response import ErrorResponse
|
||||
from paddlespeech.server.utils.errors import ErrorCode
|
||||
from paddlespeech.server.utils.errors import failed_response
|
||||
from paddlespeech.server.utils.exception import ServerBaseException
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get('/paddlespeech/asr/search/help')
|
||||
def help():
|
||||
"""help
|
||||
|
||||
Returns:
|
||||
json: the audio content search result
|
||||
"""
|
||||
response = {
|
||||
"success": "True",
|
||||
"code": 200,
|
||||
"message": {
|
||||
"global": "success"
|
||||
},
|
||||
"result": {
|
||||
"description": "acs server",
|
||||
"input": "base64 string of wavfile",
|
||||
"output": {
|
||||
"asr_result": "你好",
|
||||
"acs_result": [{
|
||||
'w': '你',
|
||||
'bg': 0.0,
|
||||
'ed': 1.2
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
return response
|
||||
|
||||
|
||||
@router.post(
|
||||
"/paddlespeech/asr/search",
|
||||
response_model=Union[ACSResponse, ErrorResponse])
|
||||
def acs(request_body: ASRRequest):
|
||||
"""acs api
|
||||
|
||||
Args:
|
||||
request_body (ASRRequest): the acs request, we reuse the http ASRRequest
|
||||
|
||||
Returns:
|
||||
json: the acs result
|
||||
"""
|
||||
try:
|
||||
# 1. get the audio data via base64 decoding
|
||||
audio_data = base64.b64decode(request_body.audio)
|
||||
|
||||
# 2. get single engine from engine pool
|
||||
engine_pool = get_engine_pool()
|
||||
acs_engine = engine_pool['acs']
|
||||
|
||||
# 3. no data stored in acs_engine, so we need to create the another instance process the data
|
||||
acs_result, asr_result = acs_engine.run(audio_data)
|
||||
|
||||
response = {
|
||||
"success": True,
|
||||
"code": 200,
|
||||
"message": {
|
||||
"description": "success"
|
||||
},
|
||||
"result": {
|
||||
"transcription": asr_result,
|
||||
"acs": acs_result
|
||||
}
|
||||
}
|
||||
|
||||
except ServerBaseException as e:
|
||||
response = failed_response(e.error_code, e.msg)
|
||||
except BaseException as e:
|
||||
response = failed_response(ErrorCode.SERVER_UNKOWN_ERR)
|
||||
logger.error(e)
|
||||
|
||||
return response
|
||||
Loading…
Reference in new issue