Jackwaterveg
4 years ago
committed by
GitHub
commit
3d5aac6a94
@ -0,0 +1,11 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
|
||||||
|
Date: 2022-1-10, Author: Jackwaterveg.
|
||||||
|
Add features to: CLI:
|
||||||
|
- Support English (librispeech/asr1/transformer).
|
||||||
|
- Support choosing `decode_method` for conformer and transformer models.
|
||||||
|
- Refactor the config, using the unified config.
|
||||||
|
- PRLink: https://github.com/PaddlePaddle/PaddleSpeech/pull/1297
|
||||||
|
|
||||||
|
***
|
||||||
@ -1,10 +1,15 @@
|
|||||||
# Speech Application based on PaddleSpeech
|
# Speech Application based on PaddleSpeech
|
||||||
|
|
||||||
|
([简体中文](./README_cn.md)|English)
|
||||||
|
|
||||||
The directory containes many speech applications in multi scenarios.
|
The directory containes many speech applications in multi scenarios.
|
||||||
|
|
||||||
* audio tagging - tag audio label in vedio
|
* audio tagging - multi-label tagging of an audio file
|
||||||
|
* automatic_video_subtitiles - generate subtitles from a video
|
||||||
* metaverse - 2D AR with TTS
|
* metaverse - 2D AR with TTS
|
||||||
* speech recogintion - vidio understanding
|
* punctuation_restoration - restore punctuation from raw text
|
||||||
|
* speech recogintion - recognize text of an audio file
|
||||||
* speech translation - end to end speech translation
|
* speech translation - end to end speech translation
|
||||||
* story talker - book reader based on OCR and TTS
|
* story talker - book reader based on OCR and TTS
|
||||||
* style_fs2 - multi style control for FastSpeech2 model
|
* style_fs2 - multi style control for FastSpeech2 model
|
||||||
|
* text_to_speech - convert text into speech
|
||||||
|
|||||||
@ -0,0 +1,128 @@
|
|||||||
|
# Customize Dataset for Audio Classification
|
||||||
|
|
||||||
|
Following this tutorial you can customize your dataset for audio classification task by using `paddlespeech` and `paddleaudio`.
|
||||||
|
|
||||||
|
A base class of classification dataset is `paddleaudio.dataset.AudioClassificationDataset`. To customize your dataset you should write a dataset class derived from `AudioClassificationDataset`.
|
||||||
|
|
||||||
|
Assuming you have some wave files that stored in your own directory. You should prepare a meta file with the information of filepaths and labels. For example the absolute path of it is `/PATH/TO/META_FILE.txt`:
|
||||||
|
```
|
||||||
|
/PATH/TO/WAVE_FILE/1.wav cat
|
||||||
|
/PATH/TO/WAVE_FILE/2.wav cat
|
||||||
|
/PATH/TO/WAVE_FILE/3.wav dog
|
||||||
|
/PATH/TO/WAVE_FILE/4.wav dog
|
||||||
|
```
|
||||||
|
Here is an example to build your custom dataset in `custom_dataset.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from paddleaudio.datasets.dataset import AudioClassificationDataset
|
||||||
|
|
||||||
|
class CustomDataset(AudioClassificationDataset):
|
||||||
|
meta_file = '/PATH/TO/META_FILE.txt'
|
||||||
|
# List all the class labels
|
||||||
|
label_list = [
|
||||||
|
'cat',
|
||||||
|
'dog',
|
||||||
|
]
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
files, labels = self._get_data()
|
||||||
|
super(CustomDataset, self).__init__(
|
||||||
|
files=files, labels=labels, feat_type='raw', **kwargs)
|
||||||
|
|
||||||
|
def _get_data(self):
|
||||||
|
'''
|
||||||
|
This method offer information of wave files and labels.
|
||||||
|
'''
|
||||||
|
files = []
|
||||||
|
labels = []
|
||||||
|
|
||||||
|
with open(self.meta_file) as f:
|
||||||
|
for line in f:
|
||||||
|
file, label_str = line.strip().split(' ')
|
||||||
|
files.append(file)
|
||||||
|
labels.append(self.label_list.index(label_str))
|
||||||
|
|
||||||
|
return files, labels
|
||||||
|
```
|
||||||
|
|
||||||
|
Then you can build dataset and data loader from `CustomDataset`:
|
||||||
|
```python
|
||||||
|
import paddle
|
||||||
|
from paddleaudio.features import LogMelSpectrogram
|
||||||
|
|
||||||
|
from custom_dataset import CustomDataset
|
||||||
|
|
||||||
|
# Feature config should be align with pretrained model
|
||||||
|
sample_rate = 32000
|
||||||
|
feat_conf = {
|
||||||
|
'sr': sample_rate,
|
||||||
|
'n_fft': 1024,
|
||||||
|
'hop_length': 320,
|
||||||
|
'window': 'hann',
|
||||||
|
'win_length': 1024,
|
||||||
|
'f_min': 50.0,
|
||||||
|
'f_max': 14000.0,
|
||||||
|
'n_mels': 64,
|
||||||
|
}
|
||||||
|
|
||||||
|
train_ds = CustomDataset(sample_rate=sample_rate)
|
||||||
|
feature_extractor = LogMelSpectrogram(**feat_conf)
|
||||||
|
|
||||||
|
train_sampler = paddle.io.DistributedBatchSampler(
|
||||||
|
train_ds, batch_size=4, shuffle=True, drop_last=False)
|
||||||
|
train_loader = paddle.io.DataLoader(
|
||||||
|
train_ds,
|
||||||
|
batch_sampler=train_sampler,
|
||||||
|
return_list=True,
|
||||||
|
use_buffer_reader=True)
|
||||||
|
```
|
||||||
|
|
||||||
|
Train model with `CustomDataset`:
|
||||||
|
```python
|
||||||
|
from paddlespeech.cls.models import cnn14
|
||||||
|
from paddlespeech.cls.models import SoundClassifier
|
||||||
|
|
||||||
|
backbone = cnn14(pretrained=True, extract_embedding=True)
|
||||||
|
model = SoundClassifier(backbone, num_class=len(train_ds.label_list))
|
||||||
|
optimizer = paddle.optimizer.Adam(
|
||||||
|
learning_rate=1e-6, parameters=model.parameters())
|
||||||
|
criterion = paddle.nn.loss.CrossEntropyLoss()
|
||||||
|
|
||||||
|
steps_per_epoch = len(train_sampler)
|
||||||
|
epochs = 10
|
||||||
|
for epoch in range(1, epochs + 1):
|
||||||
|
model.train()
|
||||||
|
|
||||||
|
for batch_idx, batch in enumerate(train_loader):
|
||||||
|
waveforms, labels = batch
|
||||||
|
# Need a padding when lengths of waveforms differ in a batch.
|
||||||
|
feats = feature_extractor(waveforms)
|
||||||
|
feats = paddle.transpose(feats, [0, 2, 1])
|
||||||
|
logits = model(feats)
|
||||||
|
loss = criterion(logits, labels)
|
||||||
|
loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
if isinstance(optimizer._learning_rate,
|
||||||
|
paddle.optimizer.lr.LRScheduler):
|
||||||
|
optimizer._learning_rate.step()
|
||||||
|
optimizer.clear_grad()
|
||||||
|
|
||||||
|
# Calculate loss
|
||||||
|
avg_loss = loss.numpy()[0]
|
||||||
|
|
||||||
|
# Calculate metrics
|
||||||
|
preds = paddle.argmax(logits, axis=1)
|
||||||
|
num_corrects = (preds == labels).numpy().sum()
|
||||||
|
num_samples = feats.shape[0]
|
||||||
|
|
||||||
|
avg_acc = num_corrects / num_samples
|
||||||
|
|
||||||
|
print_msg = 'Epoch={}/{}, Step={}/{}'.format(
|
||||||
|
epoch, epochs, batch_idx + 1, steps_per_epoch)
|
||||||
|
print_msg += ' loss={:.4f}'.format(avg_loss)
|
||||||
|
print_msg += ' acc={:.4f}'.format(avg_acc)
|
||||||
|
print_msg += ' lr={:.6f}'.format(optimizer.get_lr())
|
||||||
|
print(print_msg)
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to save the checkpoint of model and evaluate from a specific dataset, please see `paddlespeech/cli/exp/panns/train.py` for more details.
|
||||||
@ -0,0 +1,51 @@
|
|||||||
|
# Quick Start of Audio Classification
|
||||||
|
Several shell scripts provided in `./examples/esc50/cls0` will help us to quickly give it a try, for most major modules, including data preparation, model training, model evaluation, with [ESC50](ttps://github.com/karolpiczak/ESC-50) dataset.
|
||||||
|
|
||||||
|
Some of the scripts in `./examples` are not configured with GPUs. If you want to train with 8 GPUs, please modify `CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7`. If you don't have any GPU available, please set `CUDA_VISIBLE_DEVICES=` to use CPUs instead.
|
||||||
|
|
||||||
|
Let's start a audio classification task with the following steps:
|
||||||
|
|
||||||
|
- Go to the directory
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd examples/esc50/cls0
|
||||||
|
```
|
||||||
|
|
||||||
|
- Source env
|
||||||
|
```bash
|
||||||
|
source path.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
- Main entry point
|
||||||
|
```bash
|
||||||
|
CUDA_VISIBLE_DEVICES=0 ./run.sh 1
|
||||||
|
```
|
||||||
|
|
||||||
|
This demo includes fine-tuning, evaluating and deploying a audio classificatio model. More detailed information is provided in the following sections.
|
||||||
|
|
||||||
|
## Fine-tuning a model
|
||||||
|
PANNs([PANNs: Large-Scale Pretrained Audio Neural Networks for Audio Pattern Recognition](https://arxiv.org/pdf/1912.10211.pdf)) are pretrained models with [Audioset](https://research.google.com/audioset/). They can be easily used to extract audio embeddings for audio classification task.
|
||||||
|
|
||||||
|
To start a model fine-tuning, please run:
|
||||||
|
```bash
|
||||||
|
ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
|
||||||
|
feat_backend=numpy
|
||||||
|
./local/train.sh ${ngpu} ${feat_backend}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Deploy a model
|
||||||
|
Once you save a model checkpoint, you can export it to static graph and deploy by python scirpt:
|
||||||
|
|
||||||
|
- Export to a static graph
|
||||||
|
```bash
|
||||||
|
./local/export.sh ${ckpt_dir} ./export
|
||||||
|
```
|
||||||
|
The argument `ckpt_dir` should be a directory in which a model checkpoint stored, for example `checkpoint/epoch_50`.
|
||||||
|
|
||||||
|
The static graph will be exported to `./export`.
|
||||||
|
|
||||||
|
- Inference
|
||||||
|
```bash
|
||||||
|
./local/static_model_infer.sh ${infer_device} ./export ${audio_file}
|
||||||
|
```
|
||||||
|
The argument `infer_device` can be `cpu` or `gpu`, and it means which device to be used to infer. And `audio_file` should be a wave file with name `*.wav`.
|
||||||
@ -0,0 +1,42 @@
|
|||||||
|
# TTS Papers
|
||||||
|
## Text Frontend
|
||||||
|
### Polyphone
|
||||||
|
- [【g2pM】g2pM: A Neural Grapheme-to-Phoneme Conversion Package for Mandarin Chinese Based on a New Open Benchmark Dataset](https://arxiv.org/abs/2004.03136)
|
||||||
|
- [Disambiguation of Chinese Polyphones in an End-to-End Framework with Semantic Features Extracted by Pre-trained BERT](https://www1.se.cuhk.edu.hk/~hccl/publications/pub/201909_INTERSPEECH_DongyangDAI.pdf)
|
||||||
|
### Text Normalization
|
||||||
|
#### English
|
||||||
|
- [applenob/text_normalization](https://github.com/applenob/text_normalization)
|
||||||
|
### G2P
|
||||||
|
#### English
|
||||||
|
- [cmusphinx/g2p-seq2seq](https://github.com/cmusphinx/g2p-seq2seq)
|
||||||
|
|
||||||
|
## Acoustic Models
|
||||||
|
- [【AdaSpeech3】AdaSpeech 3: Adaptive Text to Speech for Spontaneous Style](https://arxiv.org/abs/2107.02530)
|
||||||
|
- [【AdaSpeech2】AdaSpeech 2: Adaptive Text to Speech with Untranscribed Data](https://arxiv.org/abs/2104.09715)
|
||||||
|
- [【AdaSpeech】AdaSpeech: Adaptive Text to Speech for Custom Voice](https://arxiv.org/abs/2103.00993)
|
||||||
|
- [【FastSpeech2】FastSpeech 2: Fast and High-Quality End-to-End Text to Speech](https://arxiv.org/abs/2006.04558)
|
||||||
|
- [【FastPitch】FastPitch: Parallel Text-to-speech with Pitch Prediction](https://arxiv.org/abs/2006.06873)
|
||||||
|
- [【SpeedySpeech】SpeedySpeech: Efficient Neural Speech Synthesis](https://arxiv.org/abs/2008.03802)
|
||||||
|
- [【FastSpeech】FastSpeech: Fast, Robust and Controllable Text to Speech](https://arxiv.org/abs/1905.09263)
|
||||||
|
- [【Transformer TTS】Neural Speech Synthesis with Transformer Network](https://arxiv.org/abs/1809.08895)
|
||||||
|
- [【Tacotron2】Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions](https://arxiv.org/abs/1712.05884)
|
||||||
|
|
||||||
|
## Vocoders
|
||||||
|
- [【RefineGAN】RefineGAN: Universally Generating Waveform Better than Ground Truth with Highly Accurate Pitch and Intensity Responses](https://arxiv.org/abs/2111.00962)
|
||||||
|
- [【Fre-GAN】Fre-GAN: Adversarial Frequency-consistent Audio Synthesis](https://arxiv.org/abs/2106.02297)
|
||||||
|
- [【StyleMelGAN】StyleMelGAN: An Efficient High-Fidelity Adversarial Vocoder with Temporal Adaptive Normalization](https://arxiv.org/abs/2011.01557)
|
||||||
|
- [【Multi-band MelGAN】Multi-band MelGAN: Faster Waveform Generation for High-Quality Text-to-Speech](https://arxiv.org/abs/2005.05106)
|
||||||
|
- [【HiFi-GAN】HiFi-GAN: Generative Adversarial Networks for Efficient and High Fidelity Speech Synthesis](https://arxiv.org/abs/2010.05646)
|
||||||
|
- [【VocGAN】VocGAN: A High-Fidelity Real-time Vocoder with a Hierarchically-nested Adversarial Network](https://arxiv.org/abs/2007.15256)
|
||||||
|
- [【Parallel WaveGAN】Parallel WaveGAN: A fast waveform generation model based on generative adversarial networks with multi-resolution spectrogram](https://arxiv.org/abs/1910.11480)
|
||||||
|
- [【MelGAN】MelGAN: Generative Adversarial Networks for Conditional Waveform Synthesis](https://arxiv.org/abs/1910.06711)
|
||||||
|
- [【WaveFlow】WaveFlow: A Compact Flow-based Model for Raw Audio](https://arxiv.org/abs/1912.01219)
|
||||||
|
- [【LPCNet】LPCNet: Improving Neural Speech Synthesis Through Linear Prediction](https://arxiv.org/abs/1810.11846)
|
||||||
|
- [【WaveRNN】Efficient Neural Audio Synthesis](https://arxiv.org/abs/1802.08435)
|
||||||
|
## GAN TTS
|
||||||
|
|
||||||
|
- [【GAN TTS】High Fidelity Speech Synthesis with Adversarial Networks](https://arxiv.org/abs/1909.11646)
|
||||||
|
|
||||||
|
## Voice Cloning
|
||||||
|
- [【SV2TTS】Transfer Learning from Speaker Verification to Multispeaker Text-to-Speech Synthesis](https://arxiv.org/abs/1806.04558)
|
||||||
|
- [【GE2E】Generalized End-to-End Loss for Speaker Verification](https://arxiv.org/abs/1710.10467)
|
||||||
@ -1,70 +1,68 @@
|
|||||||
# https://yaml.org/type/float.html
|
# https://yaml.org/type/float.html
|
||||||
data:
|
###########################################
|
||||||
train_manifest: data/manifest.train
|
# Data #
|
||||||
dev_manifest: data/manifest.dev
|
###########################################
|
||||||
test_manifest: data/manifest.test
|
train_manifest: data/manifest.train
|
||||||
min_input_len: 0.0
|
dev_manifest: data/manifest.dev
|
||||||
max_input_len: 27.0 # second
|
test_manifest: data/manifest.test
|
||||||
min_output_len: 0.0
|
min_input_len: 0.0
|
||||||
max_output_len: .inf
|
max_input_len: 27.0 # second
|
||||||
min_output_input_ratio: 0.00
|
min_output_len: 0.0
|
||||||
max_output_input_ratio: .inf
|
max_output_len: .inf
|
||||||
|
min_output_input_ratio: 0.00
|
||||||
|
max_output_input_ratio: .inf
|
||||||
|
|
||||||
collator:
|
###########################################
|
||||||
batch_size: 64 # one gpu
|
# Dataloader #
|
||||||
mean_std_filepath: data/mean_std.json
|
###########################################
|
||||||
unit_type: char
|
batch_size: 64 # one gpu
|
||||||
vocab_filepath: data/lang_char/vocab.txt
|
mean_std_filepath: data/mean_std.json
|
||||||
augmentation_config: conf/augmentation.json
|
unit_type: char
|
||||||
random_seed: 0
|
vocab_filepath: data/lang_char/vocab.txt
|
||||||
spm_model_prefix:
|
augmentation_config: conf/augmentation.json
|
||||||
spectrum_type: linear #linear, mfcc, fbank
|
random_seed: 0
|
||||||
feat_dim:
|
spm_model_prefix:
|
||||||
delta_delta: False
|
spectrum_type: linear #linear, mfcc, fbank
|
||||||
stride_ms: 10.0
|
feat_dim: 161
|
||||||
window_ms: 20.0
|
delta_delta: False
|
||||||
n_fft: None
|
stride_ms: 10.0
|
||||||
max_freq: None
|
window_ms: 20.0
|
||||||
target_sample_rate: 16000
|
n_fft: None
|
||||||
use_dB_normalization: True
|
max_freq: None
|
||||||
target_dB: -20
|
target_sample_rate: 16000
|
||||||
dither: 1.0
|
use_dB_normalization: True
|
||||||
keep_transcription_text: False
|
target_dB: -20
|
||||||
sortagrad: True
|
dither: 1.0
|
||||||
shuffle_method: batch_shuffle
|
keep_transcription_text: False
|
||||||
num_workers: 0
|
sortagrad: True
|
||||||
|
shuffle_method: batch_shuffle
|
||||||
|
num_workers: 0
|
||||||
|
|
||||||
model:
|
############################################
|
||||||
num_conv_layers: 2
|
# Network Architecture #
|
||||||
num_rnn_layers: 5
|
############################################
|
||||||
rnn_layer_size: 1024
|
num_conv_layers: 2
|
||||||
rnn_direction: forward # [forward, bidirect]
|
num_rnn_layers: 5
|
||||||
num_fc_layers: 0
|
rnn_layer_size: 1024
|
||||||
fc_layers_size_list: -1,
|
rnn_direction: forward # [forward, bidirect]
|
||||||
use_gru: False
|
num_fc_layers: 0
|
||||||
blank_id: 0
|
fc_layers_size_list: -1,
|
||||||
|
use_gru: False
|
||||||
|
blank_id: 0
|
||||||
|
|
||||||
|
|
||||||
training:
|
###########################################
|
||||||
n_epoch: 65
|
# Training #
|
||||||
accum_grad: 1
|
###########################################
|
||||||
lr: 5e-4
|
n_epoch: 65
|
||||||
lr_decay: 0.93
|
accum_grad: 1
|
||||||
weight_decay: 1e-06
|
lr: 5.0e-4
|
||||||
global_grad_clip: 3.0
|
lr_decay: 0.93
|
||||||
log_interval: 100
|
weight_decay: 1.0e-6
|
||||||
checkpoint:
|
global_grad_clip: 3.0
|
||||||
|
log_interval: 100
|
||||||
|
checkpoint:
|
||||||
kbest_n: 50
|
kbest_n: 50
|
||||||
latest_n: 5
|
latest_n: 5
|
||||||
|
|
||||||
decoding:
|
|
||||||
batch_size: 32
|
|
||||||
error_rate_type: cer
|
|
||||||
decoding_method: ctc_beam_search
|
|
||||||
lang_model_path: data/lm/zh_giga.no_cna_cmn.prune01244.klm
|
|
||||||
alpha: 2.2 #1.9
|
|
||||||
beta: 4.3
|
|
||||||
beam_size: 300
|
|
||||||
cutoff_prob: 0.99
|
|
||||||
cutoff_top_n: 40
|
|
||||||
num_proc_bsearch: 10
|
|
||||||
|
|||||||
@ -0,0 +1,10 @@
|
|||||||
|
chunk_batch_size: 32
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: ctc_beam_search
|
||||||
|
lang_model_path: data/lm/zh_giga.no_cna_cmn.prune01244.klm
|
||||||
|
alpha: 2.2 #1.9
|
||||||
|
beta: 4.3
|
||||||
|
beam_size: 300
|
||||||
|
cutoff_prob: 0.99
|
||||||
|
cutoff_top_n: 40
|
||||||
|
num_proc_bsearch: 10
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: ctc_beam_search
|
||||||
|
lang_model_path: data/lm/zh_giga.no_cna_cmn.prune01244.klm
|
||||||
|
alpha: 1.9
|
||||||
|
beta: 5.0
|
||||||
|
beam_size: 300
|
||||||
|
cutoff_prob: 0.99
|
||||||
|
cutoff_top_n: 40
|
||||||
|
num_proc_bsearch: 10
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
beam_size: 10
|
||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: attention # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||||
|
ctc_weight: 0.5 # ctc weight for attention rescoring decode mode.
|
||||||
|
decoding_chunk_size: 16 # decoding chunk size. Defaults to -1.
|
||||||
|
# <0: for decoding, use full chunk.
|
||||||
|
# >0: for decoding, use fixed chunk size as set.
|
||||||
|
# 0: used for training, it's prohibited here.
|
||||||
|
num_decoding_left_chunks: -1 # number of left chunks for decoding. Defaults to -1.
|
||||||
|
simulate_streaming: True # simulate streaming inference. Defaults to False.
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
beam_size: 10
|
||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: attention # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||||
|
ctc_weight: 0.5 # ctc weight for attention rescoring decode mode.
|
||||||
|
decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
|
||||||
|
# <0: for decoding, use full chunk.
|
||||||
|
# >0: for decoding, use fixed chunk size as set.
|
||||||
|
# 0: used for training, it's prohibited here.
|
||||||
|
num_decoding_left_chunks: -1 # number of left chunks for decoding. Defaults to -1.
|
||||||
|
simulate_streaming: False # simulate streaming inference. Defaults to False.
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: attention # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||||
|
beam_size: 10
|
||||||
|
ctc_weight: 0.5 # ctc weight for attention rescoring decode mode.
|
||||||
|
decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
|
||||||
|
# <0: for decoding, use full chunk.
|
||||||
|
# >0: for decoding, use fixed chunk size as set.
|
||||||
|
# 0: used for training, it's prohibited here.
|
||||||
|
num_decoding_left_chunks: -1 # number of left chunks for decoding. Defaults to -1.
|
||||||
|
simulate_streaming: true # simulate streaming inference. Defaults to False.
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: cer
|
||||||
|
decoding_method: attention # 'attention', 'ctc_greedy_search', 'ctc_prefix_beam_search', 'attention_rescoring'
|
||||||
|
beam_size: 10
|
||||||
|
ctc_weight: 0.5 # ctc weight for attention rescoring decode mode.
|
||||||
|
decoding_chunk_size: -1 # decoding chunk size. Defaults to -1.
|
||||||
|
# <0: for decoding, use full chunk.
|
||||||
|
# >0: for decoding, use fixed chunk size as set.
|
||||||
|
# 0: used for training, it's prohibited here.
|
||||||
|
num_decoding_left_chunks: -1 # number of left chunks for decoding. Defaults to -1.
|
||||||
|
simulate_streaming: False # simulate streaming inference. Defaults to False.
|
||||||
|
|
||||||
|
|
||||||
@ -1,85 +0,0 @@
|
|||||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
from operator import itemgetter
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import jsonlines
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
# parse config and args
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Preprocess audio and then extract features .")
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--old-dump-dir",
|
|
||||||
default=None,
|
|
||||||
type=str,
|
|
||||||
help="directory to dump feature files.")
|
|
||||||
parser.add_argument(
|
|
||||||
"--dump-dir",
|
|
||||||
type=str,
|
|
||||||
required=True,
|
|
||||||
help="directory to finetune dump feature files.")
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
old_dump_dir = Path(args.old_dump_dir).expanduser()
|
|
||||||
old_dump_dir = old_dump_dir.resolve()
|
|
||||||
dump_dir = Path(args.dump_dir).expanduser()
|
|
||||||
# use absolute path
|
|
||||||
dump_dir = dump_dir.resolve()
|
|
||||||
dump_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
assert old_dump_dir.is_dir()
|
|
||||||
assert dump_dir.is_dir()
|
|
||||||
|
|
||||||
for sub in ["train", "dev", "test"]:
|
|
||||||
# 把 old_dump_dir 里面的 *-wave.npy 软连接到 dump_dir 的对应位置
|
|
||||||
output_dir = dump_dir / sub
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
results = []
|
|
||||||
for name in os.listdir(output_dir / "raw"):
|
|
||||||
# 003918_feats.npy
|
|
||||||
utt_id = name.split("_")[0]
|
|
||||||
mel_path = output_dir / ("raw/" + name)
|
|
||||||
gen_mel = np.load(mel_path)
|
|
||||||
wave_name = utt_id + "_wave.npy"
|
|
||||||
wav = np.load(old_dump_dir / sub / ("raw/" + wave_name))
|
|
||||||
os.symlink(old_dump_dir / sub / ("raw/" + wave_name),
|
|
||||||
output_dir / ("raw/" + wave_name))
|
|
||||||
num_sample = wav.shape[0]
|
|
||||||
num_frames = gen_mel.shape[0]
|
|
||||||
wav_path = output_dir / ("raw/" + wave_name)
|
|
||||||
|
|
||||||
record = {
|
|
||||||
"utt_id": utt_id,
|
|
||||||
"num_samples": num_sample,
|
|
||||||
"num_frames": num_frames,
|
|
||||||
"feats": str(mel_path),
|
|
||||||
"wave": str(wav_path),
|
|
||||||
}
|
|
||||||
results.append(record)
|
|
||||||
|
|
||||||
results.sort(key=itemgetter("utt_id"))
|
|
||||||
|
|
||||||
with jsonlines.open(output_dir / "raw/metadata.jsonl", 'w') as writer:
|
|
||||||
for item in results:
|
|
||||||
writer.write(item)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
## Metrics
|
||||||
|
|
||||||
|
5-fold cross validation accuracy on [ESC-50](https://github.com/karolpiczak/ESC-50) dataset:
|
||||||
|
|
||||||
|
|Model|Acc|
|
||||||
|
|--|--|
|
||||||
|
|CNN14| 0.9500
|
||||||
|
|CNN10| 0.8975
|
||||||
|
|CNN6| 0.8825
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
data:
|
||||||
|
dataset: 'paddleaudio.datasets:ESC50'
|
||||||
|
num_classes: 50
|
||||||
|
train:
|
||||||
|
mode: 'train'
|
||||||
|
split: 1
|
||||||
|
dev:
|
||||||
|
mode: 'dev'
|
||||||
|
split: 1
|
||||||
|
|
||||||
|
model:
|
||||||
|
backbone: 'paddlespeech.cls.models:cnn14'
|
||||||
|
|
||||||
|
feature:
|
||||||
|
sr: 32000
|
||||||
|
n_fft: 1024
|
||||||
|
hop_length: 320
|
||||||
|
window: 'hann'
|
||||||
|
win_length: 1024
|
||||||
|
f_min: 50.0
|
||||||
|
f_max: 14000.0
|
||||||
|
n_mels: 64
|
||||||
|
|
||||||
|
training:
|
||||||
|
epochs: 50
|
||||||
|
learning_rate: 0.00005
|
||||||
|
num_workers: 2
|
||||||
|
batch_size: 16
|
||||||
|
checkpoint_dir: './checkpoint'
|
||||||
|
save_freq: 10
|
||||||
|
log_freq: 10
|
||||||
|
|
||||||
|
predicting:
|
||||||
|
audio_file: '/audio/dog.wav'
|
||||||
|
top_k: 10
|
||||||
|
checkpoint: './checkpoint/epoch_50/model.pdparams'
|
||||||
@ -1,8 +1,8 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
ckpt_dir=$1
|
ckpt=$1
|
||||||
output_dir=$2
|
output_dir=$2
|
||||||
|
|
||||||
python3 ${BIN_DIR}/export_model.py \
|
python3 ${BIN_DIR}/export_model.py \
|
||||||
--checkpoint ${ckpt_dir}/model.pdparams \
|
--checkpoint ${ckpt} \
|
||||||
--output_dir ${output_dir}
|
--output_dir ${output_dir}
|
||||||
|
|||||||
@ -1,11 +1,4 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
audio_file=$1
|
|
||||||
ckpt_dir=$2
|
|
||||||
feat_backend=$3
|
|
||||||
|
|
||||||
python3 ${BIN_DIR}/predict.py \
|
python3 ${BIN_DIR}/predict.py \
|
||||||
--wav ${audio_file} \
|
--cfg_path=$1
|
||||||
--feat_backend ${feat_backend} \
|
|
||||||
--top_k 10 \
|
|
||||||
--checkpoint ${ckpt_dir}/model.pdparams
|
|
||||||
|
|||||||
@ -1,25 +1,12 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
ngpu=$1
|
ngpu=$1
|
||||||
feat_backend=$2
|
cfg_path=$2
|
||||||
|
|
||||||
num_epochs=50
|
|
||||||
batch_size=16
|
|
||||||
ckpt_dir=./checkpoint
|
|
||||||
save_freq=10
|
|
||||||
|
|
||||||
if [ ${ngpu} -gt 0 ]; then
|
if [ ${ngpu} -gt 0 ]; then
|
||||||
python3 -m paddle.distributed.launch --gpus $CUDA_VISIBLE_DEVICES ${BIN_DIR}/train.py \
|
python3 -m paddle.distributed.launch --gpus $CUDA_VISIBLE_DEVICES ${BIN_DIR}/train.py \
|
||||||
--epochs ${num_epochs} \
|
--cfg_path ${cfg_path}
|
||||||
--feat_backend ${feat_backend} \
|
|
||||||
--batch_size ${batch_size} \
|
|
||||||
--checkpoint_dir ${ckpt_dir} \
|
|
||||||
--save_freq ${save_freq}
|
|
||||||
else
|
else
|
||||||
python3 ${BIN_DIR}/train.py \
|
python3 ${BIN_DIR}/train.py \
|
||||||
--epochs ${num_epochs} \
|
--cfg_path ${cfg_path}
|
||||||
--feat_backend ${feat_backend} \
|
|
||||||
--batch_size ${batch_size} \
|
|
||||||
--checkpoint_dir ${ckpt_dir} \
|
|
||||||
--save_freq ${save_freq}
|
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -0,0 +1,44 @@
|
|||||||
|
###########################################################
|
||||||
|
# DATA SETTING #
|
||||||
|
###########################################################
|
||||||
|
dataset_type: Ernie
|
||||||
|
train_path: data/iwslt2012_zh/train.txt
|
||||||
|
dev_path: data/iwslt2012_zh/dev.txt
|
||||||
|
test_path: data/iwslt2012_zh/test.txt
|
||||||
|
batch_size: 64
|
||||||
|
num_workers: 2
|
||||||
|
data_params:
|
||||||
|
pretrained_token: ernie-1.0
|
||||||
|
punc_path: data/iwslt2012_zh/punc_vocab
|
||||||
|
seq_len: 100
|
||||||
|
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# MODEL SETTING #
|
||||||
|
###########################################################
|
||||||
|
model_type: ErnieLinear
|
||||||
|
model:
|
||||||
|
pretrained_token: ernie-1.0
|
||||||
|
num_classes: 4
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# OPTIMIZER SETTING #
|
||||||
|
###########################################################
|
||||||
|
optimizer_params:
|
||||||
|
weight_decay: 1.0e-6 # weight decay coefficient.
|
||||||
|
|
||||||
|
scheduler_params:
|
||||||
|
learning_rate: 1.0e-5 # learning rate.
|
||||||
|
gamma: 1.0 # scheduler gamma.
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# TRAINING SETTING #
|
||||||
|
###########################################################
|
||||||
|
max_epoch: 20
|
||||||
|
num_snapshots: 5
|
||||||
|
|
||||||
|
###########################################################
|
||||||
|
# OTHER SETTING #
|
||||||
|
###########################################################
|
||||||
|
num_snapshots: 10 # max number of snapshots to keep while training
|
||||||
|
seed: 42 # random seed for paddle, random, and np.random
|
||||||
@ -1,36 +0,0 @@
|
|||||||
data:
|
|
||||||
dataset_type: Ernie
|
|
||||||
train_path: data/iwslt2012_zh/train.txt
|
|
||||||
dev_path: data/iwslt2012_zh/dev.txt
|
|
||||||
test_path: data/iwslt2012_zh/test.txt
|
|
||||||
data_params:
|
|
||||||
pretrained_token: ernie-1.0
|
|
||||||
punc_path: data/iwslt2012_zh/punc_vocab
|
|
||||||
seq_len: 100
|
|
||||||
batch_size: 64
|
|
||||||
sortagrad: True
|
|
||||||
shuffle_method: batch_shuffle
|
|
||||||
num_workers: 0
|
|
||||||
|
|
||||||
checkpoint:
|
|
||||||
kbest_n: 5
|
|
||||||
latest_n: 10
|
|
||||||
metric_type: F1
|
|
||||||
|
|
||||||
model_type: ErnieLinear
|
|
||||||
|
|
||||||
model_params:
|
|
||||||
pretrained_token: ernie-1.0
|
|
||||||
num_classes: 4
|
|
||||||
|
|
||||||
training:
|
|
||||||
n_epoch: 100
|
|
||||||
lr: !!float 1e-5
|
|
||||||
lr_decay: 1.0
|
|
||||||
weight_decay: !!float 1e-06
|
|
||||||
global_grad_clip: 5.0
|
|
||||||
log_interval: 10
|
|
||||||
log_path: log/train_ernie_linear.log
|
|
||||||
|
|
||||||
testing:
|
|
||||||
log_path: log/test_ernie_linear.log
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
#! /usr/bin/env bash
|
|
||||||
|
|
||||||
if [ $# != 2 ]; then
|
|
||||||
echo "usage: ${0} ckpt_dir avg_num"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ckpt_dir=${1}
|
|
||||||
average_num=${2}
|
|
||||||
decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
|
|
||||||
|
|
||||||
python3 -u ${BIN_DIR}/avg_model.py \
|
|
||||||
--dst_model ${decode_checkpoint} \
|
|
||||||
--ckpt_dir ${ckpt_dir} \
|
|
||||||
--num ${average_num} \
|
|
||||||
--val_best
|
|
||||||
|
|
||||||
if [ $? -ne 0 ]; then
|
|
||||||
echo "Failed in avg ckpt!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
config_path=$1
|
||||||
|
train_output_path=$2
|
||||||
|
ckpt_name=$3
|
||||||
|
text=$4
|
||||||
|
ckpt_prefix=${ckpt_name%.*}
|
||||||
|
|
||||||
|
python3 ${BIN_DIR}/punc_restore.py \
|
||||||
|
--config=${config_path} \
|
||||||
|
--checkpoint=${train_output_path}/checkpoints/${ckpt_name} \
|
||||||
|
--text=${text}
|
||||||
@ -1,26 +1,11 @@
|
|||||||
|
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
if [ $# != 2 ];then
|
|
||||||
echo "usage: ${0} config_path ckpt_path_prefix"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
|
|
||||||
echo "using $ngpu gpus..."
|
|
||||||
|
|
||||||
config_path=$1
|
config_path=$1
|
||||||
ckpt_prefix=$2
|
train_output_path=$2
|
||||||
|
ckpt_name=$3
|
||||||
python3 -u ${BIN_DIR}/test.py \
|
|
||||||
--ngpu 1 \
|
|
||||||
--config ${config_path} \
|
|
||||||
--result_file ${ckpt_prefix}.rsl \
|
|
||||||
--checkpoint_path ${ckpt_prefix}
|
|
||||||
|
|
||||||
if [ $? -ne 0 ]; then
|
ckpt_prefix=${ckpt_name%.*}
|
||||||
echo "Failed in evaluation!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
python3 ${BIN_DIR}/test.py \
|
||||||
|
--config=${config_path} \
|
||||||
|
--checkpoint=${train_output_path}/checkpoints/${ckpt_name}
|
||||||
|
|||||||
@ -1,28 +1,9 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
if [ $# != 3 ];then
|
|
||||||
echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name log_dir"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
|
|
||||||
echo "using $ngpu gpus..."
|
|
||||||
|
|
||||||
config_path=$1
|
config_path=$1
|
||||||
ckpt_name=$2
|
train_output_path=$2
|
||||||
log_dir=$3
|
|
||||||
|
|
||||||
mkdir -p exp
|
|
||||||
|
|
||||||
python3 -u ${BIN_DIR}/train.py \
|
|
||||||
--ngpu ${ngpu} \
|
|
||||||
--config ${config_path} \
|
|
||||||
--output_dir exp/${ckpt_name} \
|
|
||||||
--log_dir ${log_dir}
|
|
||||||
|
|
||||||
if [ $? -ne 0 ]; then
|
|
||||||
echo "Failed in training!"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
python3 ${BIN_DIR}/train.py \
|
||||||
|
--config=${config_path} \
|
||||||
|
--output-dir=${train_output_path} \
|
||||||
|
--ngpu=1
|
||||||
|
|||||||
@ -1,40 +1,35 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -e
|
set -e
|
||||||
|
source path.sh
|
||||||
|
|
||||||
if [ $# -ne 4 ]; then
|
gpus=0,1
|
||||||
echo "usage: bash ./run.sh stage gpu train_config avg_num"
|
stage=0
|
||||||
echo "eg: bash ./run.sh 1 0 train_config 1"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
stage=$1
|
|
||||||
stop_stage=100
|
stop_stage=100
|
||||||
gpus=$2
|
|
||||||
conf_path=$3
|
|
||||||
avg_num=$4
|
|
||||||
avg_ckpt=avg_${avg_num}
|
|
||||||
ckpt=$(basename ${conf_path} | awk -F'.' '{print $1}')
|
|
||||||
log_dir=log
|
|
||||||
|
|
||||||
source path.sh ${ckpt}
|
conf_path=conf/default.yaml
|
||||||
|
train_output_path=exp/default
|
||||||
|
ckpt_name=snapshot_iter_12840.pdz
|
||||||
|
text=今天的天气真不错啊你下午有空吗我想约你一起去吃饭
|
||||||
|
|
||||||
|
# with the following command, you can choose the stage range you want to run
|
||||||
|
# such as `./run.sh --stage 0 --stop-stage 0`
|
||||||
|
# this can not be mixed use with `$1`, `$2` ...
|
||||||
|
source ${MAIN_ROOT}/utils/parse_options.sh || exit 1
|
||||||
|
|
||||||
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
|
||||||
# prepare data
|
# prepare data
|
||||||
bash ./local/data.sh
|
./local/data.sh
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
|
||||||
# train model, all `ckpt` under `exp` dir
|
# train model, all `ckpt` under `train_output_path/checkpoints/` dir
|
||||||
CUDA_VISIBLE_DEVICES=${gpus} bash ./local/train.sh ${conf_path} ${ckpt} ${log_dir}
|
CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
|
||||||
# avg n best model
|
CUDA_VISIBLE_DEVICES=${gpus} ./local/test.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1
|
||||||
bash ./local/avg.sh exp/${ckpt}/checkpoints ${avg_num}
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
|
if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
|
||||||
# test ckpt avg_n
|
CUDA_VISIBLE_DEVICES=${gpus} ./local/punc_restore.sh ${conf_path} ${train_output_path} ${ckpt_name} ${text}|| exit -1
|
||||||
CUDA_VISIBLE_DEVICES=${gpus} bash ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1
|
|
||||||
fi
|
fi
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
decode_batch_size: 128
|
||||||
|
error_rate_type: wer
|
||||||
|
decoding_method: ctc_beam_search
|
||||||
|
lang_model_path: data/lm/common_crawl_00.prune01111.trie.klm
|
||||||
|
alpha: 1.9
|
||||||
|
beta: 0.3
|
||||||
|
beam_size: 500
|
||||||
|
cutoff_prob: 1.0
|
||||||
|
cutoff_top_n: 40
|
||||||
|
num_proc_bsearch: 8
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue